def _get_parsed_url(url):
# type: (S) -> Url
"""This is a stand-in function for `urllib3.util.parse_url`
The orignal function doesn't handle special characters very well, this simply splits
out the authentication section, creates the parsed url, then puts the authentication
section back in, bypassing validation.
:return: The new, parsed URL object
:rtype: :class:`~urllib3.util.url.Url`
"""
try:
parsed = urllib3_parse(url)
except ValueError:
scheme, _, url = url.partition("://")
auth, _, url = url.rpartition("@")
url = "{scheme}://{url}".format(scheme=scheme, url=url)
parsed = urllib3_parse(url)._replace(auth=auth)
return parsed
def vendor_artifact(ctx, package, version=None):
simple = requests.get("https://pypi.org/simple/{0}/".format(package))
pkg_str = "{0}-{1}".format(package, version)
soup = bs4.BeautifulSoup(simple.content)
links = [
a.attrs["href"] for a in soup.find_all("a") if a.getText().startswith(pkg_str)
]
for link in links:
dest_dir = _get_git_root(ctx) / "tests" / "pypi" / package
if not dest_dir.exists():
dest_dir.mkdir()
_, _, dest_path = urllib3_parse(link).path.rpartition("/")
dest_file = dest_dir / dest_path
with io.open(dest_file.as_posix(), "wb") as target_handle:
with open_file(link) as fp:
shutil.copyfileobj(fp, target_handle)
