def testInterleaved4(self):
"""tests that http server returns responses in correct order"""
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
#we do 2 requests that should take 2 seconds to complete each.
#if server/client pipelining was not working, fetching the 2 urls
#would take 4 seconds on a single connection
#if pipelining works, it should take just 2 seconds
start = time.time()
cnn.send(cnn.get('/sleep?3'))
cnn.send(cnn.get('/sleep?1'))
response1 = cnn.receive()
response2 = cnn.receive()
#we expect response1 to be returned first, because it was made first
#eventhough it takes longer
self.assertEquals('slept 3', response1.body)
self.assertEquals('slept 1', response2.body)
end = time.time()
self.assertAlmostEqual(3, end - start, places=1)
cnn.close()
def query(host, port, uri, params, timeout=20):
try:
with Timeout.push(timeout):
cnn = HTTPConnection()
cnn.connect((str(host), int(port)))
try:
request = cnn.post(str(uri), urlencode(params))
request.add_header("Content-type", "application/x-www-form-urlencoded")
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 200:
raise HTTPError("Error downloading http://%s:%s%s: %s" % (host, port, uri, response.status))
body = response.body
if response.get_header("Content-type") == "application/json":
body = json.loads(body)
return body
finally:
cnn.close()
except IOError as e:
if e.errno == 111:
raise HTTPConnectionRefused("Connection refused during downloading http://%s:%s%s" % (host, port, uri))
else:
raise HTTPError("Error downloading http://%s:%s%s: %s" % (host, port, uri, e))
except TimeoutError:
raise HTTPError("Timeout downloading http://%s:%s%s" % (host, port, uri))
def webdav_delete(self, url):
"Downloads given URL and returns it"
if url is None:
return
if type(url) == unicode:
url = url.encode("utf-8")
url_obj = urlparse.urlparse(url, "http", False)
if url_obj.scheme != "http":
self.error("Scheme '%s' is not supported", url_obj.scheme)
elif url_obj.hostname is None:
self.error("Empty hostname: %s", url)
else:
cnn = HTTPConnection()
try:
with Timeout.push(50):
port = url_obj.port
if port is None:
port = 80
cnn.connect((url_obj.hostname, port))
request = HTTPRequest()
request.method = "DELETE"
request.path = url_obj.path + url_obj.query
request.host = url_obj.hostname
request.add_header("Connection", "close")
cnn.perform(request)
except TimeoutError:
self.error("Timeout deleting %s", url)
except Exception as e:
self.error("Error deleting %s: %s", url, str(e))
finally:
try:
cnn.close()
except Exception:
pass
def upload(self, subdir, ext, content_type, data, filename=None):
host = str(random.choice(self.clconf("storage", ["storage"])))
tag = self.app().tag
id = uuid4().hex
if filename is None:
url = str("/%s/%s/%s%s/%s/%s.%s" %
(subdir, tag[0], tag[0], tag[1], tag, id, ext))
else:
url = str("/%s/%s/%s%s/%s/%s-%s" %
(subdir, tag[0], tag[0], tag[1], tag, id, filename))
uri = str("//" + host + url)
cnn = HTTPConnection()
cnn.connect((str(host), 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = url
request.host = host
request.body = data
request.add_header("Content-type", content_type)
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 201 and response.status_code != 204:
raise StaticUploadError(
self._("Error storing object {0}: {1}").format(
uri, response.status))
finally:
cnn.close()
return (uri, url, host, id)
def static_upload_zip(self, subdir, zip, upload_list):
host = str(random.choice(self.clconf("storage", ["storage"])))
id = uuid4().hex
tag = self.app().tag
url = str("/%s/%s/%s%s/%s/%s" %
(subdir, tag[0], tag[0], tag[1], tag, id))
uri = str("//" + host + url)
for ent in upload_list:
cnn = HTTPConnection()
cnn.connect((str(host), 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = str("%s/%s" % (url, ent["filename"]))
request.host = host
data = ent.get("data")
if data is None and zip:
data = zip.read(ent["zipname"])
if data is None and ent.get("path"):
with open(ent.get("path")) as f:
data = f.read()
request.body = data
request.add_header("Content-type", str(ent["content-type"]))
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 201 and response.status_code != 204:
raise StaticUploadError(
self._("Error storing object {name}: {err}").format(
name=ent["filename"], err=response.status))
finally:
cnn.close()
return uri
def static_put(self, uri, content_type, data):
if uri is None:
raise StaticUploadError(self._("Invalid store URI"))
if type(uri) == unicode:
uri = uri.encode("utf-8")
uri_obj = urlparse.urlparse(uri, "http", False)
if uri_obj.hostname is None:
raise StaticUploadError(self._("Empty hostname"))
cnn = HTTPConnection()
cnn.connect((uri_obj.hostname, 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = uri_obj.path
request.host = uri_obj.hostname
request.body = data
request.add_header("Content-type", content_type)
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 201 and response.status_code != 204:
raise StaticUploadError(
self._("Error storing object {0}: {1}").format(
uri, response.status))
finally:
cnn.close()
def upload_if_large(self, data):
if type(data) == unicode:
data = data.encode("utf-8")
if len(data) < max_inline_chunk_len:
return data
req = self.req()
url = str("/storage/%s%s/%s" % (random.choice(alphabet), random.choice(alphabet), uuid4().hex))
cnn = HTTPConnection()
try:
cnn.connect((req.host(), 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = url
request.host = req.host()
request.body = data
request.add_header("Content-type", "application/octet-stream")
request.add_header("Content-length", len(data))
response = cnn.perform(request)
if response.status_code != 201:
self.error("Error storing object: %s", response.status)
# degradation
return ""
return '<!--# include file="%s" -->' % url
finally:
cnn.close()
except IOError as e:
self.exception(e)
def testInterleaved4(self):
"""tests that http server returns responses in correct order"""
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
#we do 2 requests that should take 2 seconds to complete each.
#if server/client pipelining was not working, fetching the 2 urls
#would take 4 seconds on a single connection
#if pipelining works, it should take just 2 seconds
start = time.time()
cnn.send(cnn.get('/sleep?3'))
cnn.send(cnn.get('/sleep?1'))
response1 = cnn.receive()
response2 = cnn.receive()
#we expect response1 to be returned first, because it was made first
#eventhough it takes longer
self.assertEquals('slept 3', response1.body)
self.assertEquals('slept 1', response2.body)
end = time.time()
self.assertAlmostEqual(3, end - start, places = 1)
cnn.close()
def webdav_delete(self, url):
"Downloads given URL and returns it"
if url is None:
return
if type(url) == unicode:
url = url.encode("utf-8")
url_obj = urlparse.urlparse(url, "http", False)
if url_obj.scheme != "http":
self.error("Scheme '%s' is not supported", url_obj.scheme)
elif url_obj.hostname is None:
self.error("Empty hostname: %s", url)
else:
cnn = HTTPConnection()
try:
with Timeout.push(50):
port = url_obj.port
if port is None:
port = 80
cnn.connect((url_obj.hostname, port))
request = HTTPRequest()
request.method = "DELETE"
request.path = url_obj.path + url_obj.query
request.host = url_obj.hostname
request.add_header("Connection", "close")
cnn.perform(request)
except TimeoutError:
self.error("Timeout deleting %s", url)
except Exception as e:
self.error("Error deleting %s: %s", url, str(e))
finally:
try:
cnn.close()
except Exception:
pass
def static_upload_zip(self, subdir, zip, upload_list):
host = str(random.choice(self.clconf("storage", ["storage"])))
id = uuid4().hex
tag = self.app().tag
url = str("/%s/%s/%s%s/%s/%s" % (subdir, tag[0], tag[0], tag[1], tag, id))
uri = str("//" + host + url)
for ent in upload_list:
cnn = HTTPConnection()
cnn.connect((str(host), 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = str("%s/%s" % (url, ent["filename"]))
request.host = host
data = ent.get("data")
if data is None and zip:
data = zip.read(ent["zipname"])
if data is None and ent.get("path"):
with open(ent.get("path")) as f:
data = f.read()
request.body = data
request.add_header("Content-type", str(ent["content-type"]))
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 201 and response.status_code != 204:
raise StaticUploadError(self._("Error storing object {name}: {err}").format(name=ent["filename"], err=response.status))
finally:
cnn.close()
return uri
def do_query_service(self, service_id, svc, url, timeout, verbose=False, *args, **kwargs):
try:
with Timeout.push(timeout):
cnn = HTTPConnection()
addr = (svc.get("addr").encode("utf-8"), svc.get("port"))
try:
cnn.connect(addr)
except IOError as e:
raise ClusterError("Error connecting to service %s(%s:%s): %s" % (service_id, addr[0], addr[1], e))
params = {
"args": json.dumps(args),
"kwargs": json.dumps(kwargs)
}
try:
uri = utf2str("/service/call/%s%s" % (service_id, url))
request = cnn.post(uri, urlencode(params))
request.add_header("Content-type", "application/x-www-form-urlencoded")
request.add_header("Connection", "close")
if verbose:
self.debug("Query http://%s:%s%s", addr[0], addr[1], uri)
response = cnn.perform(request)
if response.status_code != 200:
raise ClusterError("Service %s (%s:%d) returned status %d for URL %s" % (service_id, addr[0], addr[1], response.status_code, uri))
res = json.loads(response.body)
if res.get("error"):
raise ClusterError(u"Service %s returned error: %s" % (service_id, res["error"]))
return res.get("retval")
finally:
cnn.close()
except TimeoutError:
raise ClusterError("Timeout querying %s of service %s" % (url, service_id))
def query(host, port, uri, params, timeout=20):
try:
with Timeout.push(timeout):
cnn = HTTPConnection()
cnn.connect((str(host), int(port)))
try:
request = cnn.post(str(uri), urlencode(params))
request.add_header("Content-type",
"application/x-www-form-urlencoded")
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 200:
raise HTTPError("Error downloading http://%s:%s%s: %s" %
(host, port, uri, response.status))
body = response.body
if response.get_header("Content-type") == "application/json":
body = json.loads(body)
return body
finally:
cnn.close()
except IOError as e:
if e.errno == 111:
raise HTTPConnectionRefused(
"Connection refused during downloading http://%s:%s%s" %
(host, port, uri))
else:
raise HTTPError("Error downloading http://%s:%s%s: %s" %
(host, port, uri, e))
except TimeoutError:
raise HTTPError("Timeout downloading http://%s:%s%s" %
(host, port, uri))
def upload(self, subdir, ext, content_type, data, filename=None):
host = str(random.choice(self.clconf("storage", ["storage"])))
tag = self.app().tag
id = uuid4().hex
if filename is None:
url = str("/%s/%s/%s%s/%s/%s.%s" % (subdir, tag[0], tag[0], tag[1], tag, id, ext))
else:
url = str("/%s/%s/%s%s/%s/%s-%s" % (subdir, tag[0], tag[0], tag[1], tag, id, filename))
uri = str("//" + host + url)
cnn = HTTPConnection()
cnn.connect((str(host), 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = url
request.host = host
request.body = data
request.add_header("Content-type", content_type)
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 201 and response.status_code != 204:
raise StaticUploadError(self._("Error storing object {0}: {1}").format(uri, response.status))
finally:
cnn.close()
return (uri, url, host, id)
def thermometers():
cnn = HTTPConnection()
addr = (hwserver, hwport)
cnn.connect(addr)
req = cnn.get("/temperature")
req.add_header("Connection", "close")
res = cnn.perform(req)
if res.status_code != 200:
raise RuntimeError("Could not query thermometers state: error %s" % res.status_code)
return json.loads(res.body)
def relay_set(relay, val):
# query hwserver
cnn = HTTPConnection()
addr = (hwserver, hwport)
cnn.connect(addr)
req = cnn.get(str("/relay/%s/%d" % ("on" if val else "off", relay)))
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("Connection", "close")
res = cnn.perform(req)
if res.status_code != 200:
print "Could not query relay: error %s" % res.status_code
def thermometers():
cnn = HTTPConnection()
addr = (hwserver, hwport)
cnn.connect(addr)
req = cnn.get("/temperature")
req.add_header("Connection", "close")
res = cnn.perform(req)
if res.status_code != 200:
raise RuntimeError("Could not query thermometers state: error %s" %
res.status_code)
return json.loads(res.body)
def relay_set(relay, val):
# query hwserver
cnn = HTTPConnection()
addr = (hwserver, hwport)
cnn.connect(addr)
req = cnn.get(str("/relay/%s/%d" % ("on" if val else "off", relay)))
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("Connection", "close")
res = cnn.perform(req)
if res.status_code != 200:
print "Could not query relay: error %s" % res.status_code
def main():
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
request = cnn.get('/')
response = cnn.perform(request)
print response.status
print response.headers
print response.body
cnn.close()
def main():
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
request = cnn.get('/')
response = cnn.perform(request)
print response.status
print response.headers
print response.body
cnn.close()
def fetch_post(target, id):
cnn = HTTPConnection()
try:
cnn.connect((target["host"], 80))
try:
start = time.time()
req = cnn.get("/posts/%s" % id)
res = cnn.perform(req)
if res.status_code != 200:
store_error(target, "fetch_post: %d" % res.status_code)
print "get time=%f" % (time.time() - start)
finally:
cnn.close()
except IOError as e:
store_error(target, e)
def testHTTPPost(self):
cnn = HTTPConnection()
try:
cnn.connect(('localhost', SERVER_PORT))
for i in [1, 2, 4, 8, 16, 32, 100, 1000, 10000, 100000, 200000]:
post_data = 'test post data' * i
request = cnn.post('/post', post_data, host = 'testhost.nl')
response = cnn.perform(request)
self.assertEquals('ok', response.body)
self.assertTrue('HTTP_CONTENT_LENGTH' in self.saver.environ)
self.assertEquals(len(post_data), int(self.saver.environ['HTTP_CONTENT_LENGTH']))
self.assertEquals(post_data, self.saver.body)
self.assertEquals('testhost.nl', self.saver.environ['HTTP_HOST'])
finally:
cnn.close()
def do_query_service(self,
service_id,
svc,
url,
timeout,
verbose=False,
*args,
**kwargs):
try:
with Timeout.push(timeout):
cnn = HTTPConnection()
addr = (svc.get("addr").encode("utf-8"), svc.get("port"))
try:
cnn.connect(addr)
except IOError as e:
raise ClusterError(
"Error connecting to service %s(%s:%s): %s" %
(service_id, addr[0], addr[1], e))
params = {
"args": json.dumps(args),
"kwargs": json.dumps(kwargs)
}
try:
uri = utf2str("/service/call/%s%s" % (service_id, url))
request = cnn.post(uri, urlencode(params))
request.add_header("Content-type",
"application/x-www-form-urlencoded")
request.add_header("Connection", "close")
if verbose:
self.debug("Query http://%s:%s%s", addr[0], addr[1],
uri)
response = cnn.perform(request)
if response.status_code != 200:
raise ClusterError(
"Service %s (%s:%d) returned status %d for URL %s"
% (service_id, addr[0], addr[1],
response.status_code, uri))
res = json.loads(response.body)
if res.get("error"):
raise ClusterError(u"Service %s returned error: %s" %
(service_id, res["error"]))
return res.get("retval")
finally:
cnn.close()
except TimeoutError:
raise ClusterError("Timeout querying %s of service %s" %
(url, service_id))
def delete(self):
host = str(self.get("host"))
url = str(self.get("url"))
uri = str(self.get("uri"))
cnn = HTTPConnection()
cnn.connect((str(host), 80))
try:
request = HTTPRequest()
request.method = "DELETE"
request.path = url
request.host = host
request.add_header("Connection", "close")
cnn.perform(request)
except Exception:
pass
finally:
cnn.close()
def delete(self):
host = str(self.get("host"))
url = str(self.get("url"))
uri = str(self.get("uri"))
cnn = HTTPConnection()
cnn.connect((str(host), 80))
try:
request = HTTPRequest()
request.method = "DELETE"
request.path = url
request.host = host
request.add_header("Connection", "close")
cnn.perform(request)
except Exception:
pass
finally:
cnn.close()
def testInterleaved1(self):
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
cnn.send(cnn.get('/hello/1'))
cnn.send(cnn.get('/hello/2'))
response = cnn.receive()
self.assertEquals(200, response.status_code)
chunks = list(response)
self.assertEquals('Hello World 1', chunks[0])
response = cnn.receive()
self.assertEquals(200, response.status_code)
chunks = list(response)
self.assertEquals('Hello World 2', chunks[0])
cnn.close()
def testInterleaved1(self):
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
cnn.send(cnn.get('/hello/1'))
cnn.send(cnn.get('/hello/2'))
response = cnn.receive()
self.assertEquals(200, response.status_code)
chunks = list(response)
self.assertEquals('Hello World 1', chunks[0])
response = cnn.receive()
self.assertEquals(200, response.status_code)
chunks = list(response)
self.assertEquals('Hello World 2', chunks[0])
cnn.close()
def static_delete(self, uri):
if uri is None:
raise StaticUploadError(self._("Invalid delete URI"))
if type(uri) == unicode:
uri = uri.encode("utf-8")
uri_obj = urlparse.urlparse(uri, "http", False)
if uri_obj.hostname is None:
raise StaticUploadError(self._("Empty hostname"))
cnn = HTTPConnection()
cnn.connect((uri_obj.hostname, 80))
try:
request = HTTPRequest()
request.method = "DELETE"
request.path = uri_obj.path
request.host = uri_obj.hostname
request.add_header("Connection", "close")
cnn.perform(request)
finally:
cnn.close()
def static_delete(self, uri):
if uri is None:
raise StaticUploadError(self._("Invalid delete URI"))
if type(uri) == unicode:
uri = uri.encode("utf-8")
uri_obj = urlparse.urlparse(uri, "http", False)
if uri_obj.hostname is None:
raise StaticUploadError(self._("Empty hostname"))
cnn = HTTPConnection()
cnn.connect((uri_obj.hostname, 80))
try:
request = HTTPRequest()
request.method = "DELETE"
request.path = uri_obj.path
request.host = uri_obj.hostname
request.add_header("Connection", "close")
cnn.perform(request)
finally:
cnn.close()
def testHTTPPost(self):
cnn = HTTPConnection()
try:
cnn.connect(('localhost', SERVER_PORT))
for i in [1, 2, 4, 8, 16, 32, 100, 1000, 10000, 100000, 200000]:
post_data = 'test post data' * i
request = cnn.post('/post', post_data, host='testhost.nl')
response = cnn.perform(request)
self.assertEquals('ok', response.body)
self.assertTrue('HTTP_CONTENT_LENGTH' in self.saver.environ)
self.assertEquals(
len(post_data),
int(self.saver.environ['HTTP_CONTENT_LENGTH']))
self.assertEquals(post_data, self.saver.body)
self.assertEquals('testhost.nl',
self.saver.environ['HTTP_HOST'])
finally:
cnn.close()
def comment_post(target, id, body):
content = "body=%s" % (urlencode(body))
if len(content) > 1024 * 1024:
store_error(target, "request_too_large")
return
cnn = HTTPConnection()
try:
cnn.connect((target["host"], 80))
try:
start = time.time()
req = cnn.post("/posts/%s/comments" % id, content)
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("Content-length", len(content))
res = cnn.perform(req)
if res.status_code != 302:
store_error(target, "create_comment: %d" % res.status_code)
print "comment len=%d, time=%f" % (len(content), time.time() - start)
finally:
cnn.close()
except IOError as e:
store_error(target, e)
def monitor(state):
# prepare url
url = '/monitor'
first = True
for key, val in state.iteritems():
if first:
url += '?'
first = False
else:
url += '&'
url += '%s=%s' % (key, val)
# query hwserver
cnn = HTTPConnection()
addr = (hwserver, hwport)
cnn.connect(addr)
req = cnn.get(str(url))
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("Connection", "close")
res = cnn.perform(req)
if res.status_code != 200:
raise RuntimeError("Could not query monitor: error %s" % res.status_code)
return json.loads(res.body)
def download(self, url):
"Downloads given URL and returns it"
if url is None:
raise DownloadError()
if type(url) == unicode:
url = url.encode("utf-8")
url_obj = urlparse.urlparse(url, "http", False)
if url_obj.scheme != "http":
self.error("Scheme '%s' is not supported", url_obj.scheme)
elif url_obj.hostname is None:
self.error("Empty hostname: %s", url)
else:
cnn = HTTPConnection()
try:
with Timeout.push(50):
cnn.set_limit(20000000)
port = url_obj.port
if port is None:
port = 80
cnn.connect((url_obj.hostname, port))
request = cnn.get(url_obj.path + url_obj.query)
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 200:
self.error("Error downloading %s: %s %s", url,
response.status_code, response.status)
return ""
return response.body
except TimeoutError:
self.error("Timeout downloading %s", url)
except Exception as e:
self.error("Error downloading %s: %s", url, str(e))
finally:
try:
cnn.close()
except Exception:
pass
raise DownloadError()
def monitor(state):
# prepare url
url = '/monitor'
first = True
for key, val in state.iteritems():
if first:
url += '?'
first = False
else:
url += '&'
url += '%s=%s' % (key, val)
# query hwserver
cnn = HTTPConnection()
addr = (hwserver, hwport)
cnn.connect(addr)
req = cnn.get(str(url))
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header("Connection", "close")
res = cnn.perform(req)
if res.status_code != 200:
raise RuntimeError("Could not query monitor: error %s" %
res.status_code)
return json.loads(res.body)
def static_put(self, uri, content_type, data):
if uri is None:
raise StaticUploadError(self._("Invalid store URI"))
if type(uri) == unicode:
uri = uri.encode("utf-8")
uri_obj = urlparse.urlparse(uri, "http", False)
if uri_obj.hostname is None:
raise StaticUploadError(self._("Empty hostname"))
cnn = HTTPConnection()
cnn.connect((uri_obj.hostname, 80))
try:
request = HTTPRequest()
request.method = "PUT"
request.path = uri_obj.path
request.host = uri_obj.hostname
request.body = data
request.add_header("Content-type", content_type)
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 201 and response.status_code != 204:
raise StaticUploadError(self._("Error storing object {0}: {1}").format(uri, response.status))
finally:
cnn.close()
def testInterleaved3(self):
"""tests that http client and server really support pipelining"""
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
#we do 2 requests that should take 2 seconds to complete each.
#if server/client pipelining was not working, fetching the 2 urls
#would take 4 seconds on a single connection
#if pipelining works, it should take just 2 seconds
start = time.time()
cnn.send(cnn.get('/sleep?2'))
cnn.send(cnn.get('/sleep?2'))
list(cnn.receive())
list(cnn.receive())
end = time.time()
self.assertAlmostEqual(2, end - start, places=1)
cnn.close()
def testInterleaved3(self):
"""tests that http client and server really support pipelining"""
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
#we do 2 requests that should take 2 seconds to complete each.
#if server/client pipelining was not working, fetching the 2 urls
#would take 4 seconds on a single connection
#if pipelining works, it should take just 2 seconds
start = time.time()
cnn.send(cnn.get('/sleep?2'))
cnn.send(cnn.get('/sleep?2'))
list(cnn.receive())
list(cnn.receive())
end = time.time()
self.assertAlmostEqual(2, end - start, places = 1)
cnn.close()
def testSimple(self):
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
request = cnn.get('/hello/1')
response = cnn.perform(request)
self.assertEquals(200, response.status_code)
chunks = list(response.iter)
self.assertEquals('Hello World 1', chunks[0])
request = cnn.get('/hello/2')
response = cnn.perform(request)
self.assertEquals(200, response.status_code)
body = response.body
self.assertEquals('Hello World 2', body)
request = cnn.get('/xxx')
response = cnn.perform(request)
self.assertEquals(404, response.status_code)
chunks = list(response)
self.assertEquals('Not Found', chunks[0])
cnn.close()
def testSimple(self):
cnn = HTTPConnection()
cnn.connect(('localhost', SERVER_PORT))
request = cnn.get('/hello/1')
response = cnn.perform(request)
self.assertEquals(200, response.status_code)
chunks = list(response.iter)
self.assertEquals('Hello World 1', chunks[0])
request = cnn.get('/hello/2')
response = cnn.perform(request)
self.assertEquals(200, response.status_code)
body = response.body
self.assertEquals('Hello World 2', body)
request = cnn.get('/xxx')
response = cnn.perform(request)
self.assertEquals(404, response.status_code)
chunks = list(response)
self.assertEquals('Not Found', chunks[0])
cnn.close()
def download(self, url):
"Downloads given URL and returns it"
if url is None:
raise DownloadError()
if type(url) == unicode:
url = url.encode("utf-8")
url_obj = urlparse.urlparse(url, "http", False)
if url_obj.scheme != "http":
self.error("Scheme '%s' is not supported", url_obj.scheme)
elif url_obj.hostname is None:
self.error("Empty hostname: %s", url)
else:
cnn = HTTPConnection()
try:
with Timeout.push(50):
cnn.set_limit(20000000)
port = url_obj.port
if port is None:
port = 80
cnn.connect((url_obj.hostname, port))
request = cnn.get(url_obj.path + url_obj.query)
request.add_header("Connection", "close")
response = cnn.perform(request)
if response.status_code != 200:
self.error("Error downloading %s: %s %s", url, response.status_code, response.status)
return ""
return response.body
except TimeoutError:
self.error("Timeout downloading %s", url)
except Exception as e:
self.error("Error downloading %s: %s", url, str(e))
finally:
try:
cnn.close()
except Exception:
pass
raise DownloadError()
def main():
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
request = cnn.get('/')
#you can send multiple http requests on the same connection:
cnn.send(request) #request 1
cnn.send(request) #request 2
#and receive the corresponding responses
response1 = cnn.receive()
response2 = cnn.receive()
print response1.status
print response1.headers
print response1.body
print response2.status
print response2.headers
print response2.body
cnn.close()
def testHTTPReadTimeout(self):
self.server.read_timeout = 2
cnn = HTTPConnection()
try:
cnn.connect(('localhost', SERVER_PORT))
Tasklet.sleep(1)
response = cnn.perform(cnn.get('/hello/1'))
self.assertEquals('HTTP/1.1 200 OK', response.status)
self.assertEquals('Hello World 1', response.body)
Tasklet.sleep(3)
try:
list(cnn.perform(cnn.get('/hello/2')))
self.fail('expected eof')
except HTTPError, e:
pass
except:
self.fail('expected http errror')
def testHTTPReadTimeout(self):
self.server.read_timeout = 2
cnn = HTTPConnection()
try:
cnn.connect(('localhost', SERVER_PORT))
Tasklet.sleep(1)
response = cnn.perform(cnn.get('/hello/1'))
self.assertEquals('HTTP/1.1 200 OK', response.status)
self.assertEquals('Hello World 1', response.body)
Tasklet.sleep(3)
try:
list(cnn.perform(cnn.get('/hello/2')))
self.fail('expected eof')
except HTTPError, e:
pass
except:
self.fail('expected http errror')
def main():
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
request = cnn.get('/')
#you can send multiple http requests on the same connection:
cnn.send(request) #request 1
cnn.send(request) #request 2
#and receive the corresponding responses
response1 = cnn.receive()
response2 = cnn.receive()
print response1.status
print response1.headers
print response1.body
print response2.status
print response2.headers
print response2.body
cnn.close()