def prep(self, tmpdir, data):
self.build_dir = tmpdir.join('build')
self.download_dir = tmpdir.join('download')
os.mkdir(self.build_dir)
os.mkdir(self.download_dir)
self.dist_file = "simple-1.0.tar.gz"
self.dist_file2 = "simple-2.0.tar.gz"
self.dist_path = data.packages.join(self.dist_file)
self.dist_path2 = data.packages.join(self.dist_file2)
self.dist_url = Link(path_to_url(self.dist_path))
self.dist_url2 = Link(path_to_url(self.dist_path2))
def test_unpack_http_url_with_urllib_response_without_content_type(data):
"""
It should download and unpack files even if no Content-Type header exists
"""
_real_session = PipSession()
def _fake_session_get(*args, **kwargs):
resp = _real_session.get(*args, **kwargs)
del resp.headers["Content-Type"]
return resp
session = Mock()
session.get = _fake_session_get
uri = path_to_url(data.packages.join("simple-1.0.tar.gz"))
link = Link(uri)
temp_dir = mkdtemp()
try:
unpack_http_url(
link,
temp_dir,
download_dir=None,
session=session,
)
assert set(os.listdir(temp_dir)) == {
'PKG-INFO', 'setup.cfg', 'setup.py', 'simple', 'simple.egg-info'
}
finally:
rmtree(temp_dir)
def test_unpack_file_url_thats_a_dir(self, tmpdir, data):
self.prep(tmpdir, data)
dist_path = data.packages.join("FSPkg")
dist_url = Link(path_to_url(dist_path))
unpack_file_url(dist_url, self.build_dir,
download_dir=self.download_dir)
assert os.path.isdir(os.path.join(self.build_dir, 'fspkg'))
def get_url(self, location):
url = self.run_command(
['showconfig', 'paths.default'],
show_stdout=False, cwd=location).strip()
if self._is_local_repository(url):
url = path_to_url(url)
return url.strip()
def __init__(self, url, comes_from=None, requires_python=None):
# type: (str, Optional[Union[str, HTMLPage]], Optional[str]) -> None
"""
url:
url of the resource pointed to (href of the link)
comes_from:
instance of HTMLPage where the link was found, or string.
requires_python:
String containing the `Requires-Python` metadata field, specified
in PEP 345. This may be specified by a data-requires-python
attribute in the HTML link tag, as described in PEP 503.
"""
# url can be a UNC windows share
if url.startswith('\\\\'):
url = path_to_url(url)
self.url = url
self.comes_from = comes_from
self.requires_python = requires_python if requires_python else None
super(Link, self).__init__(
key=(self.url),
defining_class=Link
)
def get_remote_url(cls, location):
urls = cls.run_command(['info'], show_stdout=False, cwd=location)
for line in urls.splitlines():
line = line.strip()
for x in ('checkout of branch: ',
'parent branch: '):
if line.startswith(x):
repo = line.split(x)[1]
if cls._is_local_repository(repo):
return path_to_url(repo)
return repo
return None
def test_hash_mismatch(self, data):
"""A hash mismatch should raise an error."""
file_url = path_to_url(
(data.packages / 'simple-1.0.tar.gz').abspath)
reqset = RequirementSet(require_hashes=True)
reqset.add_requirement(get_processed_req_from_line(
'%s --hash=sha256:badbad' % file_url, lineno=1,
))
finder = PackageFinder([data.find_links], [], session=PipSession())
resolver = self._basic_resolver(finder)
assert_raises_regexp(
HashErrors,
r'THESE PACKAGES DO NOT MATCH THE HASHES.*\n'
r' file:///.*/data/packages/simple-1\.0\.tar\.gz .*:\n'
r' Expected sha256 badbad\n'
r' Got 393043e672415891885c9a2a0929b1af95fb866d'
r'6ca016b42d2e6ce53619b653$',
resolver.resolve,
reqset)
def __init__(self, url, comes_from=None, requires_python=None):
"""
Object representing a parsed link from https://pypi.python.org/simple/*
url:
url of the resource pointed to (href of the link)
comes_from:
instance of HTMLPage where the link was found, or string.
requires_python:
String containing the `Requires-Python` metadata field, specified
in PEP 345. This may be specified by a data-requires-python
attribute in the HTML link tag, as described in PEP 503.
"""
# url can be a UNC windows share
if url.startswith('\\\\'):
url = path_to_url(url)
self.url = url
self.comes_from = comes_from
self.requires_python = requires_python if requires_python else None
def build(
self,
requirements, # type: Iterable[InstallRequirement]
session, # type: PipSession
autobuilding=False # type: bool
):
# type: (...) -> List[InstallRequirement]
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
buildset = []
format_control = self.finder.format_control
# Whether a cache directory is available for autobuilding=True.
cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir)
for req in requirements:
ephem_cache = should_use_ephemeral_cache(
req, format_control=format_control, autobuilding=autobuilding,
cache_available=cache_available,
)
if ephem_cache is None:
continue
buildset.append((req, ephem_cache))
if not buildset:
return []
# Is any wheel build not using the ephemeral cache?
if any(not ephem_cache for _, ephem_cache in buildset):
have_directory_for_build = self._wheel_dir or (
autobuilding and self.wheel_cache.cache_dir
)
assert have_directory_for_build
# TODO by @pradyunsg
# Should break up this method into 2 separate methods.
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
_cache = self.wheel_cache # shorter name
with indent_log():
build_success, build_failure = [], []
for req, ephem in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
if ephem:
output_dir = _cache.get_ephem_path_for_link(req.link)
else:
output_dir = _cache.get_path_for_link(req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req, output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(os.path.join(
req.source_dir, PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.preparer.build_dir
)
# Update the link for this.
req.link = Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link, req.source_dir, None, False,
session=session,
)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return a list of requirements that failed to build
return build_failure
def _link_for_candidate(self, link, candidate):
root = self.get_path_for_link(link)
path = os.path.join(root, candidate)
return Link(path_to_url(path))
def test_url_to_path_path_to_url_symmetry_win():
path = r'C:\tmp\file'
assert url_to_path(path_to_url(path)) == path
unc_path = r'\\unc\share\path'
assert url_to_path(path_to_url(unc_path)) == unc_path
def sort_path(path):
url = path_to_url(path)
if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
urls.append(url)
else:
files.append(url)
def build(
self,
requirements, # type: Iterable[InstallRequirement]
session, # type: PipSession
autobuilding=False # type: bool
):
# type: (...) -> List[InstallRequirement]
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
buildset = []
format_control = self.finder.format_control
# Whether a cache directory is available for autobuilding=True.
cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir)
for req in requirements:
ephem_cache = should_use_ephemeral_cache(
req,
format_control=format_control,
autobuilding=autobuilding,
cache_available=cache_available,
)
if ephem_cache is None:
continue
buildset.append((req, ephem_cache))
if not buildset:
return []
# Is any wheel build not using the ephemeral cache?
if any(not ephem_cache for _, ephem_cache in buildset):
have_directory_for_build = self._wheel_dir or (
autobuilding and self.wheel_cache.cache_dir)
assert have_directory_for_build
# TODO by @pradyunsg
# Should break up this method into 2 separate methods.
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
_cache = self.wheel_cache # shorter name
with indent_log():
build_success, build_failure = [], []
for req, ephem in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
if ephem:
output_dir = _cache.get_ephem_path_for_link(req.link)
else:
output_dir = _cache.get_path_for_link(req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req,
output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(
os.path.join(req.source_dir,
PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.preparer.build_dir)
# Update the link for this.
req.link = Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link,
req.source_dir,
None,
False,
session=session,
)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return a list of requirements that failed to build
return build_failure
def test_path_to_url_win():
assert path_to_url('c:/tmp/file') == 'file:///C:/tmp/file'
assert path_to_url('c:\\tmp\\file') == 'file:///C:/tmp/file'
assert path_to_url(r'\\unc\as\path') == 'file://unc/as/path'
path = os.path.join(os.getcwd(), 'file')
assert path_to_url('file') == 'file:' + urllib_request.pathname2url(path)
def test_url_to_path_path_to_url_symmetry_win():
path = r'C:\tmp\file'
assert url_to_path(path_to_url(path)) == path
unc_path = r'\\unc\share\path'
assert url_to_path(path_to_url(unc_path)) == unc_path
def install_req_from_line(
name, # type: str
comes_from=None, # type: Optional[Union[str, InstallRequirement]]
use_pep517=None, # type: Optional[bool]
isolated=False, # type: bool
options=None, # type: Optional[Dict[str, Any]]
wheel_cache=None, # type: Optional[WheelCache]
constraint=False # type: bool
):
# type: (...) -> InstallRequirement
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers_as_string = name.split(marker_sep, 1)
markers_as_string = markers_as_string.strip()
if not markers_as_string:
markers = None
else:
markers = Marker(markers_as_string)
else:
markers = None
name = name.strip()
req_as_string = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras_as_string = None
if is_url(name):
link = Link(name)
else:
p, extras_as_string = _strip_extras(path)
looks_like_dir = os.path.isdir(p) and (
os.path.sep in name or
(os.path.altsep is not None and os.path.altsep in name) or
name.startswith('.')
)
if looks_like_dir:
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. Neither 'setup.py' "
"nor 'pyproject.toml' found." % name
)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist',
name
)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req_as_string = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req_as_string = link.egg_fragment
# a requirement specifier
else:
req_as_string = name
if extras_as_string:
extras = Requirement("placeholder" + extras_as_string.lower()).extras
else:
extras = ()
if req_as_string is not None:
try:
req = Requirement(req_as_string)
except InvalidRequirement:
if os.path.sep in req_as_string:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req_as_string)
elif ('=' in req_as_string and
not any(op in req_as_string for op in operators)):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = ""
raise InstallationError(
"Invalid requirement: '%s'\n%s" % (req_as_string, add_msg)
)
else:
req = None
return InstallRequirement(
req, comes_from, link=link, markers=markers,
use_pep517=use_pep517, isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def sort_path(path):
url = path_to_url(path)
if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
urls.append(url)
else:
files.append(url)
def install_req_from_line(
name, # type: str
comes_from=None, # type: Optional[Union[str, InstallRequirement]]
use_pep517=None, # type: Optional[bool]
isolated=False, # type: bool
options=None, # type: Optional[Dict[str, Any]]
wheel_cache=None, # type: Optional[WheelCache]
constraint=False # type: bool
):
# type: (...) -> InstallRequirement
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers_as_string = name.split(marker_sep, 1)
markers_as_string = markers_as_string.strip()
if not markers_as_string:
markers = None
else:
markers = Marker(markers_as_string)
else:
markers = None
name = name.strip()
req_as_string = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras_as_string = None
if is_url(name):
link = Link(name)
else:
p, extras_as_string = _strip_extras(path)
looks_like_dir = os.path.isdir(p) and (os.path.sep in name or
(os.path.altsep is not None
and os.path.altsep in name)
or name.startswith('.'))
if looks_like_dir:
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. Neither 'setup.py' "
"nor 'pyproject.toml' found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist', name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req_as_string = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req_as_string = link.egg_fragment
# a requirement specifier
else:
req_as_string = name
if extras_as_string:
extras = Requirement("placeholder" + extras_as_string.lower()).extras
else:
extras = ()
if req_as_string is not None:
try:
req = Requirement(req_as_string)
except InvalidRequirement:
if os.path.sep in req_as_string:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req_as_string)
elif ('=' in req_as_string
and not any(op in req_as_string for op in operators)):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = ""
raise InstallationError("Invalid requirement: '%s'\n%s" %
(req_as_string, add_msg))
else:
req = None
return InstallRequirement(
req,
comes_from,
link=link,
markers=markers,
use_pep517=use_pep517,
isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def parse_editable(editable_req):
# type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]]
"""Parses an editable requirement into:
- a requirement name
- an URL
- extras
- editable options
Accepted requirements:
svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
.[some_extra]
"""
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
if os.path.isdir(url_no_extras):
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
msg = ('File "setup.py" not found. Directory cannot be installed '
'in editable mode: {}'.format(
os.path.abspath(url_no_extras)))
pyproject_path = make_pyproject_path(url_no_extras)
if os.path.isfile(pyproject_path):
msg += ('\n(A "pyproject.toml" file was found, but editable '
'mode currently requires a setup.py based build.)')
raise InstallationError(msg)
# Treating it as code that has already been checked out
url_no_extras = path_to_url(url_no_extras)
if url_no_extras.lower().startswith('file:'):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
Requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, None
for version_control in vcs:
if url.lower().startswith('%s:' % version_control):
url = '%s+%s' % (version_control, url)
break
if '+' not in url:
raise InstallationError(
'%s should either be a path to a local project or a VCS url '
'beginning with svn+, git+, hg+, or bzr+' % editable_req)
vc_type = url.split('+', 1)[0].lower()
if not vcs.get_backend(vc_type):
error_message = 'For --editable=%s only ' % editable_req + \
', '.join([backend.name + '+URL' for backend in vcs.backends]) + \
' is currently supported'
raise InstallationError(error_message)
package_name = Link(url).egg_fragment
if not package_name:
raise InstallationError(
"Could not detect requirement name for '%s', please specify one "
"with #egg=your_package_name" % editable_req)
return package_name, url, None
def from_line(cls,
name,
comes_from=None,
isolated=False,
options=None,
wheel_cache=None,
constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip._internal.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = Marker(markers)
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
looks_like_dir = os.path.isdir(p) and (os.path.sep in name or
(os.path.altsep is not None
and os.path.altsep in name)
or name.startswith('.'))
if looks_like_dir:
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist', name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
if extras:
extras = Requirement("placeholder" + extras.lower()).extras
else:
extras = ()
if req is not None:
try:
req = Requirement(req)
except InvalidRequirement:
if os.path.sep in req:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req)
elif '=' in req and not any(op in req for op in operators):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = traceback.format_exc()
raise InstallationError("Invalid requirement: '%s'\n%s" %
(req, add_msg))
return cls(
req,
comes_from,
link=link,
markers=markers,
isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def parse_editable(editable_req):
"""Parses an editable requirement into:
- a requirement name
- an URL
- extras
- editable options
Accepted requirements:
svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
.[some_extra]
"""
from pip._internal.index import Link
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
if os.path.isdir(url_no_extras):
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' not found." %
url_no_extras)
# Treating it as code that has already been checked out
url_no_extras = path_to_url(url_no_extras)
if url_no_extras.lower().startswith('file:'):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
Requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, None
for version_control in vcs:
if url.lower().startswith('%s:' % version_control):
url = '%s+%s' % (version_control, url)
break
if '+' not in url:
raise InstallationError(
'%s should either be a path to a local project or a VCS url '
'beginning with svn+, git+, hg+, or bzr+' % editable_req)
vc_type = url.split('+', 1)[0].lower()
if not vcs.get_backend(vc_type):
error_message = 'For --editable=%s only ' % editable_req + \
', '.join([backend.name + '+URL' for backend in vcs.backends]) + \
' is currently supported'
raise InstallationError(error_message)
package_name = Link(url).egg_fragment
if not package_name:
raise InstallationError(
"Could not detect requirement name for '%s', please specify one "
"with #egg=your_package_name" % editable_req)
return _strip_postfix(package_name), url, None
def build(self, requirements, session, autobuilding=False):
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
from pip._internal import index
building_is_possible = self._wheel_dir or (autobuilding and
self.wheel_cache.cache_dir)
assert building_is_possible
buildset = []
for req in requirements:
if req.constraint:
continue
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.',
req.name,
)
elif autobuilding and req.editable:
pass
elif autobuilding and not req.source_dir:
pass
elif autobuilding and req.link and not req.link.is_artifact:
# VCS checkout. Build wheel just for this run.
buildset.append((req, True))
else:
ephem_cache = False
if autobuilding:
link = req.link
base, ext = link.splitext()
if index.egg_info_matches(base, None, link) is None:
# E.g. local directory. Build wheel just for this run.
ephem_cache = True
if "binary" not in index.fmt_ctl_formats(
self.finder.format_control,
canonicalize_name(req.name)):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.",
req.name,
)
continue
buildset.append((req, ephem_cache))
if not buildset:
return True
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
_cache = self.wheel_cache # shorter name
with indent_log():
build_success, build_failure = [], []
for req, ephem in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
if ephem:
output_dir = _cache.get_ephem_path_for_link(req.link)
else:
output_dir = _cache.get_path_for_link(req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req,
output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(
os.path.join(req.source_dir,
PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.preparer.build_dir)
# Update the link for this.
req.link = index.Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link,
req.source_dir,
None,
False,
session=session,
)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
SPACE.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
SPACE.join([req.name for req in build_failure]),
)
# Return True if all builds were successful
return len(build_failure) == 0
def build(self, requirements, session, autobuilding=False):
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
from pip._internal import index
building_is_possible = self._wheel_dir or (
autobuilding and self.wheel_cache.cache_dir
)
assert building_is_possible
buildset = []
for req in requirements:
if req.constraint:
continue
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.', req.name,
)
elif autobuilding and req.editable:
pass
elif autobuilding and not req.source_dir:
pass
elif autobuilding and req.link and not req.link.is_artifact:
# VCS checkout. Build wheel just for this run.
buildset.append((req, True))
else:
ephem_cache = False
if autobuilding:
link = req.link
base, ext = link.splitext()
if index.egg_info_matches(base, None, link) is None:
# E.g. local directory. Build wheel just for this run.
ephem_cache = True
if "binary" not in index.fmt_ctl_formats(
self.finder.format_control,
canonicalize_name(req.name)):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.", req.name,
)
continue
buildset.append((req, ephem_cache))
if not buildset:
return True
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
_cache = self.wheel_cache # shorter name
with indent_log():
build_success, build_failure = [], []
for req, ephem in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
if ephem:
output_dir = _cache.get_ephem_path_for_link(req.link)
else:
output_dir = _cache.get_path_for_link(req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req, output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(os.path.join(
req.source_dir, PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.preparer.build_dir
)
# Update the link for this.
req.link = index.Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link, req.source_dir, None, False,
session=session,
)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return True if all builds were successful
return len(build_failure) == 0
def parse_editable(editable_req):
"""Parses an editable requirement into:
- a requirement name
- an URL
- extras
- editable options
Accepted requirements:
svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
.[some_extra]
"""
from pip._internal.index import Link
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
if os.path.isdir(url_no_extras):
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' not found." %
url_no_extras
)
# Treating it as code that has already been checked out
url_no_extras = path_to_url(url_no_extras)
if url_no_extras.lower().startswith('file:'):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
Requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, None
for version_control in vcs:
if url.lower().startswith('%s:' % version_control):
url = '%s+%s' % (version_control, url)
break
if '+' not in url:
raise InstallationError(
'%s should either be a path to a local project or a VCS url '
'beginning with svn+, git+, hg+, or bzr+' %
editable_req
)
vc_type = url.split('+', 1)[0].lower()
if not vcs.get_backend(vc_type):
error_message = 'For --editable=%s only ' % editable_req + \
', '.join([backend.name + '+URL' for backend in vcs.backends]) + \
' is currently supported'
raise InstallationError(error_message)
package_name = Link(url).egg_fragment
if not package_name:
raise InstallationError(
"Could not detect requirement name for '%s', please specify one "
"with #egg=your_package_name" % editable_req
)
return _strip_postfix(package_name), url, None
def _link_for_candidate(self, link, candidate):
root = self.get_path_for_link(link)
path = os.path.join(root, candidate)
return Link(path_to_url(path))
def from_line(
cls, name, comes_from=None, isolated=False, options=None,
wheel_cache=None, constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip._internal.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = Marker(markers)
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
looks_like_dir = os.path.isdir(p) and (
os.path.sep in name or
(os.path.altsep is not None and os.path.altsep in name) or
name.startswith('.')
)
if looks_like_dir:
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name
)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist',
name
)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
if extras:
extras = Requirement("placeholder" + extras.lower()).extras
else:
extras = ()
if req is not None:
try:
req = Requirement(req)
except InvalidRequirement:
if os.path.sep in req:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req)
elif '=' in req and not any(op in req for op in operators):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = traceback.format_exc()
raise InstallationError(
"Invalid requirement: '%s'\n%s" % (req, add_msg))
return cls(
req, comes_from, link=link, markers=markers,
isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def parse_editable(editable_req):
# type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]]
"""Parses an editable requirement into:
- a requirement name
- an URL
- extras
- editable options
Accepted requirements:
svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
.[some_extra]
"""
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
if os.path.isdir(url_no_extras):
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
msg = (
'File "setup.py" not found. Directory cannot be installed '
'in editable mode: {}'.format(os.path.abspath(url_no_extras))
)
pyproject_path = make_pyproject_path(url_no_extras)
if os.path.isfile(pyproject_path):
msg += (
'\n(A "pyproject.toml" file was found, but editable '
'mode currently requires a setup.py based build.)'
)
raise InstallationError(msg)
# Treating it as code that has already been checked out
url_no_extras = path_to_url(url_no_extras)
if url_no_extras.lower().startswith('file:'):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
Requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, None
for version_control in vcs:
if url.lower().startswith('%s:' % version_control):
url = '%s+%s' % (version_control, url)
break
if '+' not in url:
raise InstallationError(
'%s should either be a path to a local project or a VCS url '
'beginning with svn+, git+, hg+, or bzr+' %
editable_req
)
vc_type = url.split('+', 1)[0].lower()
if not vcs.get_backend(vc_type):
error_message = 'For --editable=%s only ' % editable_req + \
', '.join([backend.name + '+URL' for backend in vcs.backends]) + \
' is currently supported'
raise InstallationError(error_message)
package_name = Link(url).egg_fragment
if not package_name:
raise InstallationError(
"Could not detect requirement name for '%s', please specify one "
"with #egg=your_package_name" % editable_req
)
return package_name, url, None
def test_path_to_url_unix():
assert path_to_url('/tmp/file') == 'file:///tmp/file'
path = os.path.join(os.getcwd(), 'file')
assert path_to_url('file') == 'file://' + urllib_request.pathname2url(path)
def test_path_to_url_win():
assert path_to_url('c:/tmp/file') == 'file:///C:/tmp/file'
assert path_to_url('c:\\tmp\\file') == 'file:///C:/tmp/file'
assert path_to_url(r'\\unc\as\path') == 'file://unc/as/path'
path = os.path.join(os.getcwd(), 'file')
assert path_to_url('file') == 'file:' + urllib_request.pathname2url(path)
def test_path_to_url_unix():
assert path_to_url('/tmp/file') == 'file:///tmp/file'
path = os.path.join(os.getcwd(), 'file')
assert path_to_url('file') == 'file://' + urllib_request.pathname2url(path)