def __init__(self, url):
"""Construct a :class:`Link` from a url.
:param url: A string-like object representing a url.
"""
purl = urlparse.urlparse(url)
if purl.scheme == '' or (WINDOWS and len(purl.scheme)
== 1): # This is likely a drive letter.
purl = urlparse.urlparse(self._normalize(url))
self._url = purl
def __init__(self, url):
"""Construct a :class:`Link` from a url.
:param url: A string-like object representing a url.
"""
purl = urlparse.urlparse(url)
if purl.scheme == '' or (
WINDOWS and len(purl.scheme) == 1): # This is likely a drive letter.
purl = urlparse.urlparse(self._normalize(url))
self._url = purl
def resolve(
self,
line, # type: LogicalLine
origin, # type: str
is_constraints=False, # type: bool
fetcher=None, # type: Optional[URLFetcher]
):
# type: (...) -> Iterator[Source]
def create_parse_error(msg):
# type: (str) -> ParseError
return ParseError(
line,
"Problem resolving {} file: {}".format(
"constraints" if is_constraints else "requirements", msg),
)
url = urlparse.urlparse(urlparse.urljoin(self.origin, origin))
if url.scheme and url.netloc:
if fetcher is None:
raise create_parse_error(
"The source is a url but no fetcher was supplied to resolve its contents with."
)
try:
with self.from_url(fetcher, origin) as source:
yield source
except OSError as e:
raise create_parse_error(str(e))
return
path = url.path if url.scheme == "file" else origin
try:
with self.from_file(path, is_constraints=is_constraints) as source:
yield source
except (IOError, OSError) as e:
raise create_parse_error(str(e))
def maybe_trust_insecure_host(url):
url_info = urlparse.urlparse(url)
if 'http' == url_info.scheme:
# Implicitly trust explicitly asked for http indexes and find_links repos instead of
# requiring seperate trust configuration.
trusted_hosts.append(url_info.netloc)
return url
def vcs(cls, url, rewrite=True):
result = urlparse.urlparse(url)
fragment_params = urlparse.parse_qs(result.fragment)
values = fragment_params.get('egg')
if not values or len(values) != 1:
raise ValueError('Expected the vcs requirement url to have an #egg=<name> fragment. '
'Got: {}'.format(url))
# N.B.: Constraints do not work for vcs urls.
return cls(key=values[0], requirement=url, rewrite=rewrite, constrain=False)
def _fixup_install(
dist, # type: str
install_dir, # type: str
):
# type: (...) -> None
# The dist-info metadata directory is named as specifed in:
# https://www.python.org/dev/peps/pep-0427/
# https://packaging.python.org/specifications/recording-installed-packages/#the-dist-info-directory
project_name_and_version = dist_metadata.project_name_and_version(
dist) or ProjectNameAndVersion.from_filename(dist)
project_name = safe_name(
project_name_and_version.project_name).replace("-", "_")
version = safe_version(project_name_and_version.version).replace(
"-", "_")
dist_info_dir = os.path.join(
install_dir,
"{project_name}-{version}.dist-info".format(
project_name=project_name, version=version),
)
# The `direct_url.json` file is both mandatory for Pip to install and non-hermetic for
# Pex's purpose, since it contains the absolute local filesystem path to any local wheel
# file Pex installs via Pip. We remove the file and its entry in RECORD so that PEX files
# are bytewise reproducible. The absence of the direct_url.json file only affects Pex venvs
# where further mutation by PEP-compatible packaging tooling (e.g.: Pip) may be hindered.
# In particular, `pip freeze` for any distributions provided by local projects or archives
# will produce unuseful entries for those distributions.
#
# See:
# https://www.python.org/dev/peps/pep-0610/
# https://packaging.python.org/specifications/direct-url/#specification
direct_url_relpath = os.path.join(os.path.basename(dist_info_dir),
"direct_url.json")
direct_url_abspath = os.path.join(os.path.dirname(dist_info_dir),
direct_url_relpath)
if not os.path.exists(direct_url_abspath):
return
with open(direct_url_abspath) as fp:
if urlparse.urlparse(json.load(fp)["url"]).scheme != "file":
return
os.unlink(direct_url_abspath)
# The RECORD is a csv file with the path to each installed file in the 1st column.
# See: https://www.python.org/dev/peps/pep-0376/#record
with closing(
fileinput.input(files=[os.path.join(dist_info_dir, "RECORD")],
inplace=True)) as record_fi:
for line in record_fi:
if line.split(",")[0] != direct_url_relpath:
# N.B.: These lines include the newline already.
sys.stdout.write(line)
def open_source():
url = urlparse.urlparse(location)
if url.scheme and url.netloc:
if fetcher is None:
raise ValueError(
"The location is a url but no fetcher was supplied to resolve its contents "
"with."
)
return Source.from_url(fetcher=fetcher, url=location, is_constraints=is_constraints)
path = url.path if url.scheme == "file" else location
return Source.from_file(path=path, is_constraints=is_constraints)
def _parse_requirement_line(
line, # type: LogicalLine
basepath=None, # type: Optional[str]
):
# type: (...) -> ReqInfo
basepath = basepath or os.getcwd()
editable, processed_text = _strip_requirement_options(line)
# Handle urls (Pip proprietary).
parsed_url = urlparse.urlparse(processed_text)
if _is_recognized_pip_url_scheme(parsed_url.scheme):
project_name, marker = _try_parse_fragment_project_name_and_marker(
parsed_url.fragment)
if not project_name:
project_name = _try_parse_project_name_from_path(parsed_url.path)
url = parsed_url._replace(fragment="").geturl()
return ReqInfo.create(line,
project_name=project_name,
url=url,
marker=marker,
editable=editable)
# Handle local archives and project directories (Pip proprietary).
maybe_abs_path, marker = _try_parse_pip_local_formats(processed_text,
basepath=basepath)
if maybe_abs_path is not None and any(
os.path.isfile(os.path.join(maybe_abs_path, *p))
for p in ((), ("setup.py", ), ("pyproject.toml", ))):
archive_or_project_path = os.path.realpath(maybe_abs_path)
is_local_project = os.path.isdir(archive_or_project_path)
project_name = (
None if is_local_project else
_try_parse_project_name_from_path(archive_or_project_path))
return ReqInfo.create(
line,
project_name=project_name,
url=archive_or_project_path,
marker=marker,
editable=editable,
is_local_project=is_local_project,
)
# Handle PEP-440. See: https://www.python.org/dev/peps/pep-0440.
#
# The `pkg_resources.Requirement.parse` method does all of this for us (via
# `packaging.requirements.Requirement`) except for the handling of PEP-440 direct url
# references; so we strip those urls out first.
requirement, direct_reference_url = _split_direct_references(
processed_text)
try:
req = Requirement.parse(requirement)
return ReqInfo.create(
line,
project_name=req.name,
url=direct_reference_url or req.url,
marker=req.marker,
editable=editable,
)
except RequirementParseError as e:
raise ParseError(
line, "Problem parsing {!r} as a requirement: {}".format(
processed_text, e))
def _parse_requirement_line(
line, # type: LogicalLine
basepath=None, # type: Optional[str]
):
# type: (...) -> ParsedRequirement
basepath = basepath or os.getcwd()
editable, processed_text = _strip_requirement_options(line)
project_name, direct_reference_url = _split_direct_references(processed_text)
parsed_url = urlparse.urlparse(direct_reference_url or processed_text)
# Handle non local URLs (Pip proprietary).
if _is_recognized_non_local_pip_url_scheme(parsed_url.scheme):
project_name_extras_and_marker = _try_parse_fragment_project_name_and_marker(
parsed_url.fragment
)
project_name, extras, marker = (
project_name_extras_and_marker
if project_name_extras_and_marker
else (project_name, None, None)
)
specifier = None # type: Optional[SpecifierSet]
if not project_name:
project_name_and_specifier = _try_parse_project_name_and_specifier_from_path(
parsed_url.path
)
if project_name_and_specifier is not None:
project_name = project_name_and_specifier.project_name
specifier = project_name_and_specifier.specifier
if project_name is None:
raise ParseError(
line,
(
"Could not determine a project name for URL requirement {}, consider using "
"#egg=<project name>."
),
)
url = parsed_url._replace(fragment="").geturl()
requirement = parse_requirement_from_project_name_and_specifier(
project_name,
extras=extras,
specifier=specifier,
marker=marker,
)
return URLRequirement.create(line, url, requirement, editable=editable)
# Handle local archives and project directories via path or file URL (Pip proprietary).
local_requirement = parsed_url._replace(scheme="").geturl()
project_name_extras_and_marker = _try_parse_pip_local_formats(
local_requirement, basepath=basepath
)
maybe_abs_path, extras, marker = (
project_name_extras_and_marker
if project_name_extras_and_marker
else (project_name, None, None)
)
if maybe_abs_path is not None and any(
os.path.isfile(os.path.join(maybe_abs_path, *p))
for p in ((), ("setup.py",), ("pyproject.toml",))
):
archive_or_project_path = os.path.realpath(maybe_abs_path)
if os.path.isdir(archive_or_project_path):
return LocalProjectRequirement.create(
line,
archive_or_project_path,
extras=extras,
marker=marker,
editable=editable,
)
try:
requirement = parse_requirement_from_dist(
archive_or_project_path, extras=extras, marker=marker
)
return URLRequirement.create(
line, archive_or_project_path, requirement, editable=editable
)
except dist_metadata.UnrecognizedDistributionFormat:
# This is not a recognized local archive distribution. Fall through and try parsing as a
# PEP-440 requirement.
pass
# Handle PEP-440. See: https://www.python.org/dev/peps/pep-0440.
#
# The `pkg_resources.Requirement.parse` method does all of this for us (via
# `packaging.requirements.Requirement`) except for the handling of PEP-440 direct url
# references; which we handled above and won't encounter here.
try:
return PyPIRequirement.create(line, Requirement.parse(processed_text), editable=editable)
except RequirementParseError as e:
raise ParseError(
line, "Problem parsing {!r} as a requirement: {}".format(processed_text, e)
)
