def _parse_editable(editable_req):
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
if os.path.isdir(url_no_extras):
if not os.path.exists(os.path.join(url_no_extras, "setup.py")):
raise PipError(
"Directory %r is not installable. File 'setup.py' not found." %
url_no_extras)
# Treating it as code that has already been checked out
url_no_extras = _path_to_url(url_no_extras)
if url_no_extras.lower().startswith("file:"):
return _parse_local_package_name(
url_no_extras[len("file://"):]), url_no_extras
if "+" not in url:
raise PipError(
"%s should either be a path to a local project or a VCS url "
"beginning with svn+, git+, hg+, or bzr+" % editable_req)
package_name = _egg_fragment(url)
if not package_name:
raise PipError(
"Could not detect requirement name for '%s', please specify one "
"with #egg=your_package_name" % editable_req)
return package_name, url
def _parse_requirement_url(req_str):
original_req_str = req_str
# Some requirements lines begin with a `git+` or similar to indicate the VCS. If this is the
# case, remove this before proceeding any further.
for v in VCS_SCHEMES:
if req_str.startswith(v + "+"):
req_str = req_str[len(v) + 1:]
break
# Strip out the marker temporarily while we parse out any potential URLs
marker_sep = "; " if _is_url(req_str) else ";"
marker_str = None
link = None
if ";" in req_str:
req_str, marker_str = req_str.split(marker_sep, 1)
if _is_url(req_str):
link = Link(req_str)
else:
path = os.path.normpath(os.path.abspath(req_str))
p, _ = _strip_extras(path)
url = _get_url_from_path(p, req_str)
if url is not None:
link = Link(url)
# it's a local file, dir, or url
if link is not None:
# Handle relative file URLs
if link.scheme == "file" and re.search(r"\.\./", link.url):
link = Link(
_path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel_info = WHEEL_FILE_RE.match(link.filename)
if wheel_info is None:
raise PipError(f"Invalid wheel name: {link.filename}")
wheel_name = wheel_info.group("name").replace("_", "-")
wheel_version = wheel_info.group("ver").replace("_", "-")
req_str = f"{wheel_name}=={wheel_version}"
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req_str = _egg_fragment(link.url)
if req_str is None:
raise PipError(
f"Missing egg fragment in URL: {original_req_str}")
req_str = f"{req_str}@{link.url}"
# Reassemble the requirement string with the original marker
if marker_str is not None:
req_str = f"{req_str}{marker_sep}{marker_str}"
return req_str
def installed_distributions(
local: bool = False,
paths: List[os.PathLike] = []) -> Dict[str, Distribution]:
# Check whether our version of pip supports the `--path` parameter
if pip_api.PIP_VERSION < parse("19.2") and paths:
raise PipError(
f"pip {pip_api.PIP_VERSION} does not support the `paths` argument")
if pip_api.PIP_VERSION < parse("9.0.0"):
return _old_installed_distributions(local)
return _new_installed_distributions(local, paths)
def parse_requirements(filename, options=None, include_invalid=False):
to_parse = {filename}
parsed = set()
name_to_req = {}
while to_parse:
filename = to_parse.pop()
dirname = os.path.dirname(filename)
parsed.add(filename)
# Combine multi-line commands
lines = "".join(_read_file(filename)).replace("\\\n", "").splitlines()
lines_enum = enumerate(lines, 1)
lines_enum = _ignore_comments(lines_enum)
lines_enum = _skip_regex(lines_enum, options)
for lineno, line in lines_enum:
req = None
known, _ = parser.parse_known_args(line.strip().split())
if known.req:
try: # Try to parse this as a requirement specification
req = requirements.Requirement(known.req)
except requirements.InvalidRequirement:
try:
_check_invalid_requirement(known.req)
except PipError as e:
if include_invalid:
req = UnparsedRequirement(known.req, str(e),
filename, lineno)
else:
raise
elif known.requirements:
full_path = os.path.join(dirname, known.requirements)
if full_path not in parsed:
to_parse.add(full_path)
elif known.editable:
name, url = _parse_editable(known.editable)
req = requirements.Requirement("%s @ %s" % (name, url))
else:
pass # This is an invalid requirement
# If we've found a requirement, add it
if req:
if not isinstance(req, UnparsedRequirement):
req.comes_from = "-r {} (line {})".format(filename, lineno)
if req.name not in name_to_req:
name_to_req[req.name.lower()] = req
else:
raise PipError(
"Double requirement given: %s (already in %s, name=%r)"
% (req, name_to_req[req.name], req.name))
return name_to_req
def _check_invalid_requirement(req):
if os.path.sep in req:
add_msg = "It looks like a path."
if os.path.exists(req):
add_msg += " It does exist."
else:
add_msg += " File '%s' does not exist." % (req)
elif "=" in req and not any(op in req for op in operators):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = traceback.format_exc()
raise PipError("Invalid requirement: '%s'\n%s" % (req, add_msg))
def _parse_local_package_name(path):
"""Tokenize setup.py and walk the syntax tree to find the package name"""
try:
with open(os.path.join(path, "setup.py")) as f:
tree = ast.parse(f.read())
setup_kwargs = [
expr.value.keywords for expr in tree.body
if isinstance(expr, ast.Expr) and isinstance(expr.value, ast.Call)
and expr.value.func.id == "setup"
][0]
value = [kw.value for kw in setup_kwargs if kw.arg == "name"][0]
return value.s
except (IndexError, AttributeError, IOError, OSError):
raise PipError("Directory %r is not installable. "
"Could not parse package name from 'setup.py'." % path)
def _get_url_from_path(path, name):
if _looks_like_path(name) and os.path.isdir(path):
if _is_installable_dir(path):
return _path_to_url(path)
# TODO: The is_installable_dir test here might not be necessary
# now that it is done in load_pyproject_toml too.
raise PipError(
f"Directory {name!r} is not installable. Neither 'setup.py' "
"nor 'pyproject.toml' found.")
if not _is_archive_file(path):
return None
if os.path.isfile(path):
return _path_to_url(path)
urlreq_parts = name.split("@", 1)
if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]):
# If the path contains '@' and the part before it does not look
# like a path, try to treat it as a PEP 440 URL req instead.
return None
return _path_to_url(path)
def parse_requirements(
filename: os.PathLike,
options: Optional[Any] = None,
include_invalid: bool = False,
strict_hashes: bool = False,
) -> Dict[str, Union[Requirement, UnparsedRequirement]]:
to_parse = {filename}
parsed = set()
name_to_req = {}
while to_parse:
filename = to_parse.pop()
dirname = os.path.dirname(filename)
parsed.add(filename)
# Combine multi-line commands
lines = "".join(_read_file(filename)).replace("\\\n", "").splitlines()
lines_enum = enumerate(lines, 1)
lines_enum = _ignore_comments(lines_enum)
lines_enum = _skip_regex(lines_enum, options)
for lineno, line in lines_enum:
req: Optional[Union[Requirement, UnparsedRequirement]] = None
known, _ = parser.parse_known_args(line.strip().split())
hashes_by_kind = defaultdict(list)
if known.hashes:
for hsh in known.hashes:
kind, hsh = hsh.split(":", 1)
if kind not in VALID_HASHES:
raise PipError(
"Invalid --hash kind %s, expected one of %s" %
(kind, VALID_HASHES))
hashes_by_kind[kind].append(hsh)
if known.req:
req_str = str().join(known.req)
try:
parsed_req_str = _parse_requirement_url(req_str)
except PipError as e:
if include_invalid:
req = UnparsedRequirement(req_str, str(e), filename,
lineno)
else:
raise
try: # Try to parse this as a requirement specification
if req is None:
req = Requirement(
parsed_req_str,
hashes=dict(hashes_by_kind),
filename=filename,
lineno=lineno,
)
except requirements.InvalidRequirement:
try:
_check_invalid_requirement(req_str)
except PipError as e:
if include_invalid:
req = UnparsedRequirement(req_str, str(e),
filename, lineno)
else:
raise
elif known.requirement:
full_path = os.path.join(dirname, known.requirement)
if full_path not in parsed:
to_parse.add(full_path)
elif known.editable:
name, url = _parse_editable(known.editable)
req = Requirement(
"%s @ %s" % (name, url),
filename=filename,
lineno=lineno,
editable=True,
)
else:
pass # This is an invalid requirement
# If we've found a requirement, add it
if req:
if not isinstance(req, UnparsedRequirement):
req.comes_from = "-r {} (line {})".format(
filename, lineno) # type: ignore
if req.marker is not None and not req.marker.evaluate():
continue
if req.name not in name_to_req:
name_to_req[req.name.lower()] = req
else:
raise PipError(
"Double requirement given: %s (already in %s, name=%r)"
% (req, name_to_req[req.name], req.name))
if strict_hashes:
missing_hashes = [
req for req in name_to_req.values() if not req.hashes
]
if len(missing_hashes) > 0:
raise PipError(
"Missing hashes for requirement in %s, line %s" %
(missing_hashes[0].filename, missing_hashes[0].lineno))
return name_to_req