def _get_parsed_url(url): # type: (S) -> Url """This is a stand-in function for `urllib3.util.parse_url` The orignal function doesn't handle special characters very well, this simply splits out the authentication section, creates the parsed url, then puts the authentication section back in, bypassing validation. :return: The new, parsed URL object :rtype: :class:`~urllib3.util.url.Url` """ try: parsed = urllib3_parse(url) except ValueError: scheme, _, url = url.partition("://") auth, _, url = url.rpartition("@") url = "{scheme}://{url}".format(scheme=scheme, url=url) parsed = urllib3_parse(url)._replace(auth=auth) return parsed
def vendor_artifact(ctx, package, version=None): simple = requests.get("https://pypi.org/simple/{0}/".format(package)) pkg_str = "{0}-{1}".format(package, version) soup = bs4.BeautifulSoup(simple.content) links = [ a.attrs["href"] for a in soup.find_all("a") if a.getText().startswith(pkg_str) ] for link in links: dest_dir = _get_git_root(ctx) / "tests" / "pypi" / package if not dest_dir.exists(): dest_dir.mkdir() _, _, dest_path = urllib3_parse(link).path.rpartition("/") dest_file = dest_dir / dest_path with io.open(dest_file.as_posix(), "wb") as target_handle: with open_file(link) as fp: shutil.copyfileobj(fp, target_handle)