def test_download_parts(): fpath = '/tmp/_test_download' fpath_part = '/tmp/_test_download_part' part_size = 400000 ua = UserAgent(max_retries=3) if not os.path.exists(fpath) or os.path.getsize(fpath) < part_size: ua.download(dl_url, fpath) assert os.path.getsize(fpath) > part_size with open(fpath_part, 'w') as chunk: chunk.write(open(fpath).read(part_size)) chunk.flush() assert part_size == os.path.getsize(fpath_part) try: r = ua.download(dl_url, fpath_part, resume=True) except RetriesExceeded: print "Redirect failed" except ConnectionError as e: print "Not found: %s %s" % (type(e).__name__, e) else: assert len(r) + part_size == os.path.getsize(fpath) assert os.path.getsize(fpath) == os.path.getsize(fpath_part) assert filecmp.cmp(fpath, fpath_part) print "Resuming download finished successful" os.remove(fpath) os.remove(fpath_part)
def test_unicode_post(): byte_string = b'\xc8\xb9\xc8\xbc\xc9\x85' unicode_string = byte_string.decode('utf-8') headers = {'CONTENT_LENGTH': str(len(byte_string)), 'CONTENT_TYPE': 'text/plain; charset=utf-8'} with wsgiserver(check_upload(byte_string, headers)): useragent = UserAgent() useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=unicode_string)
def test_final_url(): with wsgiserver(normal_response()): useragent = UserAgent() url = 'http://127.0.0.1:54323/' response = useragent.urlopen(url) assert url == response.final_url assert 200 == response.status_code
def test_server_error_with_unicode(): with wsgiserver(internal_server_error()): useragent = UserAgent() with pytest.raises(BadStatusCode): useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=u"12345")
def test_redirect(): redirect = 'http://127.0.0.1:54323/redirected' with wsgiserver(check_redirect(redirect)): useragent = UserAgent() response = useragent.urlopen('http://127.0.0.1:54323/') assert redirect == response.final_url assert b"redirected" == response.read()
def test_error_handling(): ua = UserAgent(max_retries=1) try: 1 / 0 except ZeroDivisionError as err: err.trace = sys.exc_info()[2] with pytest.raises(ZeroDivisionError) as cm: #@UndefinedVariable ua._handle_error(err) assert str(cm.traceback[-1]).strip().endswith('1 / 0')
def test_open_multiple_domains(): ua = UserAgent(max_retries=1) for domain in ('google.com', 'microsoft.com'): try: r = ua.urlopen('http://' + domain + '/') except RetriesExceeded: print "Redirect failed" else: print r.headers
def test_open_multiple_domains_parallel(): ua = UserAgent(max_retries=1, headers=DEFAULT_HEADERS) domains = 'google.com', 'microsoft.com', 'debian.org', 'spiegel.de', 'heise.de' get_domain_headers = lambda d: (d, ua.urlopen('http://' + d).headers) gp = gevent.pool.Group() for domain, hdr in gp.imap_unordered(get_domain_headers, domains): print domain print hdr print
def test_bytes_post(): headers = { 'CONTENT_LENGTH': '5', 'CONTENT_TYPE': 'application/octet-stream' } with wsgiserver(check_upload(b"12345", headers)): useragent = UserAgent() useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=b"12345")
def test_gzip(): ua = UserAgent(max_retries=1, headers=DEFAULT_HEADERS) resp = ua.urlopen('https://google.com') assert resp.headers.get('content-encoding') == 'gzip' cl = int(resp.headers.get('content-length', 0)) if cl: # Looks like google dropped content-length recently assert cl > 5000 assert len(resp.content) > 2 * cl # Check, if unzip produced readable output for word in ('doctype', 'html', 'function', 'script', 'google'): assert word in resp.content
def test_file_post(): body = tempfile.NamedTemporaryFile("a+b", delete=False) name = body.name try: body.write(b"123456789") body.close() headers = {'CONTENT_LENGTH': '9', 'CONTENT_TYPE': 'application/octet-stream'} with wsgiserver(check_upload(b"123456789", headers)): useragent = UserAgent() with open(name, 'rb') as body: useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=body) finally: os.remove(name)
def test_server_error_with_file(): body = tempfile.NamedTemporaryFile("a+b", delete=False) name = body.name try: body.write(b"123456789") body.close() with wsgiserver(internal_server_error()): useragent = UserAgent() with pytest.raises(BadStatusCode): with open(name, 'rb') as body: useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=body) finally: os.remove(name)
def test_download(): fpath = '/tmp/_test_download' if os.path.exists(fpath): os.remove(fpath) ua = UserAgent(max_retries=3) try: r = ua.download(dl_url, fpath) except RetriesExceeded: print "Redirect failed" except ConnectionError as e: print "Not found: %s %s" % (type(e).__name__, e) else: fl = os.path.getsize(fpath) cl = r.headers.get('Content-Length') cl = int(cl) if cl else None assert cl == fl len_str = 'OK' if cl == fl else 'CL: %s / FL: %s' % (cl, fl) print "Download finished:", len_str
def test_multipart_post(): body = tempfile.NamedTemporaryFile("a+b", delete=False) name = body.name try: body.write(b"123456789") headers = { 'CONTENT_LENGTH': '237', 'CONTENT_TYPE': 'multipart/form-data; boundary=custom_boundary' } files = { 'file': ('report.xls', body, 'application/vnd.ms-excel', { 'Expires': '0' }, 'custom_boundary') } with wsgiserver( check_upload(( b'--custom_boundary\r\n' b'Content-Disposition: form-data; name="files"\r\n' b'\r\n' b'file\r\n' b'--custom_boundary\r\n' b'Content-Disposition: form-data; name="file"; filename="report.xls"\r\n' b'Content-Type: application/vnd.ms-excel\r\n' b'Expires: 0\r\n' b'\r\n' b'\r\n' b'--custom_boundary--' b'\r\n'), headers)): useragent = UserAgent() useragent.urlopen('http://127.0.0.1:54323/', method='POST', files=files) finally: body.close() os.remove(name)
def test_cookiejar(): with wsgiserver(set_cookie()): useragent = UserAgent(cookiejar=CookieJar()) assert b"" == useragent.urlopen('http://127.0.0.1:54323/').read()
def test_bytes_post(): headers = {'CONTENT_LENGTH': '5', 'CONTENT_TYPE': 'application/octet-stream'} with wsgiserver(check_upload(b"12345", headers)): useragent = UserAgent() useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=b"12345")
def test_redirect(): with wsgiserver(check_redirect()): useragent = UserAgent() assert b"redirected" == useragent.urlopen('http://127.0.0.1:54323/').read()
def test_brotli_response(): with wsgiserver(return_brotli()): resp = UserAgent().urlopen('http://127.0.0.1:54323/', params={"path": "/"}) assert resp.status_code == 200 assert resp.content == b"https://github.com/gwik/geventhttpclient"
def test_params(): with wsgiserver(check_querystring()): resp = UserAgent().urlopen('http://127.0.0.1:54323/?param1=b', params={"param2":"hello"}) assert resp.status_code == 200 assert resp.content == b"param1=b¶m2=hello"
def test_params_quoted(): with wsgiserver(check_querystring()): resp = UserAgent().urlopen('http://127.0.0.1:54323/?a/b', params={"path":"/"}) assert resp.status_code == 200 assert resp.content == b"a/b&path=%2F"
def test_redirect(): with wsgiserver(check_redirect()): resp = UserAgent().urlopen('http://127.0.0.1:54323/') assert resp.status_code == 200 assert b"redirected" == resp.content
def test_redirect(): with wsgiserver(check_redirect()): useragent = UserAgent() assert b"redirected" == useragent.urlopen( 'http://127.0.0.1:54323/').read()