def test_download_parts():
fpath = '/tmp/_test_download'
fpath_part = '/tmp/_test_download_part'
part_size = 400000
ua = UserAgent(max_retries=3)
if not os.path.exists(fpath) or os.path.getsize(fpath) < part_size:
ua.download(dl_url, fpath)
assert os.path.getsize(fpath) > part_size
with open(fpath_part, 'w') as chunk:
chunk.write(open(fpath).read(part_size))
chunk.flush()
assert part_size == os.path.getsize(fpath_part)
try:
r = ua.download(dl_url, fpath_part, resume=True)
except RetriesExceeded:
print "Redirect failed"
except ConnectionError as e:
print "Not found: %s %s" % (type(e).__name__, e)
else:
assert len(r) + part_size == os.path.getsize(fpath)
assert os.path.getsize(fpath) == os.path.getsize(fpath_part)
assert filecmp.cmp(fpath, fpath_part)
print "Resuming download finished successful"
os.remove(fpath)
os.remove(fpath_part)
def test_unicode_post():
byte_string = b'\xc8\xb9\xc8\xbc\xc9\x85'
unicode_string = byte_string.decode('utf-8')
headers = {'CONTENT_LENGTH': str(len(byte_string)), 'CONTENT_TYPE': 'text/plain; charset=utf-8'}
with wsgiserver(check_upload(byte_string, headers)):
useragent = UserAgent()
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=unicode_string)
def test_final_url():
with wsgiserver(normal_response()):
useragent = UserAgent()
url = 'http://127.0.0.1:54323/'
response = useragent.urlopen(url)
assert url == response.final_url
assert 200 == response.status_code
def test_server_error_with_unicode():
with wsgiserver(internal_server_error()):
useragent = UserAgent()
with pytest.raises(BadStatusCode):
useragent.urlopen('http://127.0.0.1:54323/',
method='POST',
payload=u"12345")
def test_redirect():
redirect = 'http://127.0.0.1:54323/redirected'
with wsgiserver(check_redirect(redirect)):
useragent = UserAgent()
response = useragent.urlopen('http://127.0.0.1:54323/')
assert redirect == response.final_url
assert b"redirected" == response.read()
def test_unicode_post():
byte_string = b'\xc8\xb9\xc8\xbc\xc9\x85'
unicode_string = byte_string.decode('utf-8')
headers = {'CONTENT_LENGTH': str(len(byte_string)), 'CONTENT_TYPE': 'text/plain; charset=utf-8'}
with wsgiserver(check_upload(byte_string, headers)):
useragent = UserAgent()
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=unicode_string)
def test_download_parts():
fpath = '/tmp/_test_download'
fpath_part = '/tmp/_test_download_part'
part_size = 400000
ua = UserAgent(max_retries=3)
if not os.path.exists(fpath) or os.path.getsize(fpath) < part_size:
ua.download(dl_url, fpath)
assert os.path.getsize(fpath) > part_size
with open(fpath_part, 'w') as chunk:
chunk.write(open(fpath).read(part_size))
chunk.flush()
assert part_size == os.path.getsize(fpath_part)
try:
r = ua.download(dl_url, fpath_part, resume=True)
except RetriesExceeded:
print "Redirect failed"
except ConnectionError as e:
print "Not found: %s %s" % (type(e).__name__, e)
else:
assert len(r) + part_size == os.path.getsize(fpath)
assert os.path.getsize(fpath) == os.path.getsize(fpath_part)
assert filecmp.cmp(fpath, fpath_part)
print "Resuming download finished successful"
os.remove(fpath)
os.remove(fpath_part)
def test_error_handling():
ua = UserAgent(max_retries=1)
try:
1 / 0
except ZeroDivisionError as err:
err.trace = sys.exc_info()[2]
with pytest.raises(ZeroDivisionError) as cm: #@UndefinedVariable
ua._handle_error(err)
assert str(cm.traceback[-1]).strip().endswith('1 / 0')
def test_open_multiple_domains():
ua = UserAgent(max_retries=1)
for domain in ('google.com', 'microsoft.com'):
try:
r = ua.urlopen('http://' + domain + '/')
except RetriesExceeded:
print "Redirect failed"
else:
print r.headers
def test_open_multiple_domains_parallel():
ua = UserAgent(max_retries=1, headers=DEFAULT_HEADERS)
domains = 'google.com', 'microsoft.com', 'debian.org', 'spiegel.de', 'heise.de'
get_domain_headers = lambda d: (d, ua.urlopen('http://' + d).headers)
gp = gevent.pool.Group()
for domain, hdr in gp.imap_unordered(get_domain_headers, domains):
print domain
print hdr
print
def test_error_handling():
ua = UserAgent(max_retries=1)
try:
1 / 0
except ZeroDivisionError as err:
err.trace = sys.exc_info()[2]
with pytest.raises(ZeroDivisionError) as cm: #@UndefinedVariable
ua._handle_error(err)
assert str(cm.traceback[-1]).strip().endswith('1 / 0')
def test_open_multiple_domains():
ua = UserAgent(max_retries=1)
for domain in ('google.com', 'microsoft.com'):
try:
r = ua.urlopen('http://' + domain + '/')
except RetriesExceeded:
print "Redirect failed"
else:
print r.headers
def test_open_multiple_domains_parallel():
ua = UserAgent(max_retries=1, headers=DEFAULT_HEADERS)
domains = 'google.com', 'microsoft.com', 'debian.org', 'spiegel.de', 'heise.de'
get_domain_headers = lambda d: (d, ua.urlopen('http://' + d).headers)
gp = gevent.pool.Group()
for domain, hdr in gp.imap_unordered(get_domain_headers, domains):
print domain
print hdr
print
def test_bytes_post():
headers = {
'CONTENT_LENGTH': '5',
'CONTENT_TYPE': 'application/octet-stream'
}
with wsgiserver(check_upload(b"12345", headers)):
useragent = UserAgent()
useragent.urlopen('http://127.0.0.1:54323/',
method='POST',
payload=b"12345")
def test_gzip():
ua = UserAgent(max_retries=1, headers=DEFAULT_HEADERS)
resp = ua.urlopen('https://google.com')
assert resp.headers.get('content-encoding') == 'gzip'
cl = int(resp.headers.get('content-length', 0))
if cl:
# Looks like google dropped content-length recently
assert cl > 5000
assert len(resp.content) > 2 * cl
# Check, if unzip produced readable output
for word in ('doctype', 'html', 'function', 'script', 'google'):
assert word in resp.content
def test_gzip():
ua = UserAgent(max_retries=1, headers=DEFAULT_HEADERS)
resp = ua.urlopen('https://google.com')
assert resp.headers.get('content-encoding') == 'gzip'
cl = int(resp.headers.get('content-length', 0))
if cl:
# Looks like google dropped content-length recently
assert cl > 5000
assert len(resp.content) > 2 * cl
# Check, if unzip produced readable output
for word in ('doctype', 'html', 'function', 'script', 'google'):
assert word in resp.content
def test_file_post():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
body.close()
headers = {'CONTENT_LENGTH': '9', 'CONTENT_TYPE': 'application/octet-stream'}
with wsgiserver(check_upload(b"123456789", headers)):
useragent = UserAgent()
with open(name, 'rb') as body:
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=body)
finally:
os.remove(name)
def test_file_post():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
body.close()
headers = {'CONTENT_LENGTH': '9', 'CONTENT_TYPE': 'application/octet-stream'}
with wsgiserver(check_upload(b"123456789", headers)):
useragent = UserAgent()
with open(name, 'rb') as body:
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=body)
finally:
os.remove(name)
def test_server_error_with_file():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
body.close()
with wsgiserver(internal_server_error()):
useragent = UserAgent()
with pytest.raises(BadStatusCode):
with open(name, 'rb') as body:
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=body)
finally:
os.remove(name)
def test_server_error_with_file():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
body.close()
with wsgiserver(internal_server_error()):
useragent = UserAgent()
with pytest.raises(BadStatusCode):
with open(name, 'rb') as body:
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=body)
finally:
os.remove(name)
def test_download():
fpath = '/tmp/_test_download'
if os.path.exists(fpath):
os.remove(fpath)
ua = UserAgent(max_retries=3)
try:
r = ua.download(dl_url, fpath)
except RetriesExceeded:
print "Redirect failed"
except ConnectionError as e:
print "Not found: %s %s" % (type(e).__name__, e)
else:
fl = os.path.getsize(fpath)
cl = r.headers.get('Content-Length')
cl = int(cl) if cl else None
assert cl == fl
len_str = 'OK' if cl == fl else 'CL: %s / FL: %s' % (cl, fl)
print "Download finished:", len_str
def test_download():
fpath = '/tmp/_test_download'
if os.path.exists(fpath):
os.remove(fpath)
ua = UserAgent(max_retries=3)
try:
r = ua.download(dl_url, fpath)
except RetriesExceeded:
print "Redirect failed"
except ConnectionError as e:
print "Not found: %s %s" % (type(e).__name__, e)
else:
fl = os.path.getsize(fpath)
cl = r.headers.get('Content-Length')
cl = int(cl) if cl else None
assert cl == fl
len_str = 'OK' if cl == fl else 'CL: %s / FL: %s' % (cl, fl)
print "Download finished:", len_str
def test_multipart_post():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
headers = {
'CONTENT_LENGTH': '237',
'CONTENT_TYPE': 'multipart/form-data; boundary=custom_boundary'
}
files = {
'file': ('report.xls', body, 'application/vnd.ms-excel', {
'Expires': '0'
}, 'custom_boundary')
}
with wsgiserver(
check_upload((
b'--custom_boundary\r\n'
b'Content-Disposition: form-data; name="files"\r\n'
b'\r\n'
b'file\r\n'
b'--custom_boundary\r\n'
b'Content-Disposition: form-data; name="file"; filename="report.xls"\r\n'
b'Content-Type: application/vnd.ms-excel\r\n'
b'Expires: 0\r\n'
b'\r\n'
b'\r\n'
b'--custom_boundary--'
b'\r\n'), headers)):
useragent = UserAgent()
useragent.urlopen('http://127.0.0.1:54323/',
method='POST',
files=files)
finally:
body.close()
os.remove(name)
def test_cookiejar():
with wsgiserver(set_cookie()):
useragent = UserAgent(cookiejar=CookieJar())
assert b"" == useragent.urlopen('http://127.0.0.1:54323/').read()
def test_bytes_post():
headers = {'CONTENT_LENGTH': '5', 'CONTENT_TYPE': 'application/octet-stream'}
with wsgiserver(check_upload(b"12345", headers)):
useragent = UserAgent()
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=b"12345")
def test_redirect():
with wsgiserver(check_redirect()):
useragent = UserAgent()
assert b"redirected" == useragent.urlopen('http://127.0.0.1:54323/').read()
def test_server_error_with_unicode():
with wsgiserver(internal_server_error()):
useragent = UserAgent()
with pytest.raises(BadStatusCode):
useragent.urlopen('http://127.0.0.1:54323/', method='POST', payload=u"12345")
def test_brotli_response():
with wsgiserver(return_brotli()):
resp = UserAgent().urlopen('http://127.0.0.1:54323/',
params={"path": "/"})
assert resp.status_code == 200
assert resp.content == b"https://github.com/gwik/geventhttpclient"
def test_params():
with wsgiserver(check_querystring()):
resp = UserAgent().urlopen('http://127.0.0.1:54323/?param1=b', params={"param2":"hello"})
assert resp.status_code == 200
assert resp.content == b"param1=b¶m2=hello"
def test_params_quoted():
with wsgiserver(check_querystring()):
resp = UserAgent().urlopen('http://127.0.0.1:54323/?a/b', params={"path":"/"})
assert resp.status_code == 200
assert resp.content == b"a/b&path=%2F"
def test_redirect():
with wsgiserver(check_redirect()):
resp = UserAgent().urlopen('http://127.0.0.1:54323/')
assert resp.status_code == 200
assert b"redirected" == resp.content
def test_redirect():
with wsgiserver(check_redirect()):
useragent = UserAgent()
assert b"redirected" == useragent.urlopen(
'http://127.0.0.1:54323/').read()
