def __init__(self,
timeout=sock._GLOBAL_DEFAULT_TIMEOUT,
filters=None,
follow_redirect=False,
force_follow_redirect=False,
max_follow_redirect=MAX_FOLLOW_REDIRECTS,
pool_instance=None,
response_class=None,
**ssl_args):
""" HttpConnection constructor
:param timeout: socket timeout
:param filters: list, list of http filters. see the doc of http filters
for more info
:param follow_redirect: boolean, by default is false. If true,
if the HTTP status is 301, 302 or 303 the client will follow
the location.
:param max_follow_redirect: max number of redirection. If max is reached
the RedirectLimit exception is raised.
:param pool_instance: a pool instance inherited from
`restkit.pool.PoolInterface`
:param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
for more information.
"""
self._sock = None
self.timeout = timeout
self.headers = []
self.req_headers = []
self.ua = USER_AGENT
self.url = None
self.follow_redirect = follow_redirect
self.nb_redirections = max_follow_redirect
self.force_follow_redirect = force_follow_redirect
self.method = 'GET'
self.body = None
self.response_body = StringIO()
self.final_url = None
# build filter lists
self.filters = Filters(filters)
self.ssl_args = ssl_args or {}
if not pool_instance:
self.should_close = True
self.pool = None
else:
self.pool = pool_instance
self.should_close = False
if response_class is not None:
self.response_class = response_class
def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT,
filters=None, follow_redirect=False, force_follow_redirect=False,
max_follow_redirect=MAX_FOLLOW_REDIRECTS,
pool_instance=None, response_class=None,
**ssl_args):
""" HttpConnection constructor
:param timeout: socket timeout
:param filters: list, list of http filters. see the doc of http filters
for more info
:param follow_redirect: boolean, by default is false. If true,
if the HTTP status is 301, 302 or 303 the client will follow
the location.
:param max_follow_redirect: max number of redirection. If max is reached
the RedirectLimit exception is raised.
:param pool_instance: a pool instance inherited from
`restkit.pool.PoolInterface`
:param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
for more information.
"""
self._sock = None
self.timeout = timeout
self.headers = []
self.req_headers = []
self.ua = USER_AGENT
self.url = None
self.follow_redirect = follow_redirect
self.nb_redirections = max_follow_redirect
self.force_follow_redirect = force_follow_redirect
self.method = 'GET'
self.body = None
self.response_body = StringIO()
self.final_url = None
# build filter lists
self.filters = Filters(filters)
self.ssl_args = ssl_args or {}
if not pool_instance:
self.should_close = True
self.pool = None
else:
self.pool = pool_instance
self.should_close = False
if response_class is not None:
self.response_class = response_class
class HttpConnection(object):
""" Http Connection object. """
version = (1, 1)
response_class = HttpResponse
def __init__(self, timeout=sock._GLOBAL_DEFAULT_TIMEOUT,
filters=None, follow_redirect=False, force_follow_redirect=False,
max_follow_redirect=MAX_FOLLOW_REDIRECTS,
pool_instance=None, response_class=None,
**ssl_args):
""" HttpConnection constructor
:param timeout: socket timeout
:param filters: list, list of http filters. see the doc of http filters
for more info
:param follow_redirect: boolean, by default is false. If true,
if the HTTP status is 301, 302 or 303 the client will follow
the location.
:param max_follow_redirect: max number of redirection. If max is reached
the RedirectLimit exception is raised.
:param pool_instance: a pool instance inherited from
`restkit.pool.PoolInterface`
:param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
for more information.
"""
self._sock = None
self.timeout = timeout
self.headers = []
self.req_headers = []
self.ua = USER_AGENT
self.url = None
self.follow_redirect = follow_redirect
self.nb_redirections = max_follow_redirect
self.force_follow_redirect = force_follow_redirect
self.method = 'GET'
self.body = None
self.response_body = StringIO()
self.final_url = None
# build filter lists
self.filters = Filters(filters)
self.ssl_args = ssl_args or {}
if not pool_instance:
self.should_close = True
self.pool = None
else:
self.pool = pool_instance
self.should_close = False
if response_class is not None:
self.response_class = response_class
def make_connection(self):
""" initate a connection if needed or reuse a socket"""
# apply on connect filters
self.filters.apply("on_connect", self)
if self._sock is not None:
return self._sock
addr = (self.host, self.port)
s = None
# if we defined a pool use it
if self.pool is not None:
s = self.pool.get(addr)
if not s:
# pool is empty or we don't use a pool
if self.uri.scheme == "https":
s = sock.connect(addr, True, self.timeout, **self.ssl_args)
else:
s = sock.connect(addr, False, self.timeout)
return s
def clean_connections(self):
sock.close(self._sock)
if hasattr(self.pool,'clear'):
self.pool.clear_host((self.host, self.port))
def release_connection(self, address, socket):
if not self.pool:
return
self.pool.put(address, self._sock)
def parse_url(self, url):
""" parse url and get host/port"""
self.uri = urlparse.urlparse(url)
if self.uri.scheme not in ('http', 'https'):
raise InvalidUrl("None valid url")
host, port = util.parse_netloc(self.uri)
self.host = host
self.port = port
def set_body(self, body, content_type=None, content_length=None,
chunked=False):
""" set HTTP body and manage form if needed """
if not body:
if content_type is not None:
self.headers.append(('Content-Type', content_type))
if self.method in ('POST', 'PUT'):
self.headers.append(("Content-Length", "0"))
return
# set content lengh if needed
if isinstance(body, dict):
if content_type is not None and \
content_type.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(content_type)
boundary = opts.get('boundary', uuid.uuid4().hex)
body, self.headers = multipart_form_encode(body,
self.headers, boundary)
else:
content_type = "application/x-www-form-urlencoded; charset=utf-8"
body = form_encode(body)
elif hasattr(body, "boundary"):
content_type = "multipart/form-data; boundary=%s" % body.boundary
content_length = body.get_size()
if not content_type:
content_type = 'application/octet-stream'
if hasattr(body, 'name'):
content_type = mimetypes.guess_type(body.name)[0]
if not content_length:
if hasattr(body, 'fileno'):
try:
body.flush()
except IOError:
pass
content_length = str(os.fstat(body.fileno())[6])
elif hasattr(body, 'getvalue'):
try:
content_length = str(len(body.getvalue()))
except AttributeError:
pass
elif isinstance(body, types.StringTypes):
body = util.to_bytestring(body)
content_length = len(body)
if content_length:
self.headers.append(("Content-Length", content_length))
if content_type is not None:
self.headers.append(('Content-Type', content_type))
elif not chunked:
raise RequestError("Can't determine content length and" +
"Transfer-Encoding header is not chunked")
self.body = body
def request(self, url, method='GET', body=None, headers=None):
""" make effective request
:param url: str, url string
:param method: str, by default GET. http verbs
:param body: the body, could be a string, an iterator or a file-like object
:param headers: dict or list of tupple, http headers
"""
self._sock = None
self.url = url
self.final_url = url
self.parse_url(url)
self.method = method.upper()
self.headers = []
# headers are better as list
headers = headers or []
if isinstance(headers, dict):
headers = list(headers.items())
ua = USER_AGENT
content_length = None
accept_encoding = 'identity'
chunked = False
content_type = None
# default host
try:
host = self.uri.netloc.encode('ascii')
except UnicodeEncodeError:
host = self.uri.netloc.encode('idna')
# normalize headers
for name, value in headers:
name = name.title()
if name == "User-Agent":
ua = value
elif name == "Content-Type":
content_type = value
elif name == "Content-Length":
content_length = str(value)
elif name == "Accept-Encoding":
accept_encoding = value
elif name == "Host":
host = value
elif name == "Transfer-Encoding":
if value.lower() == "chunked":
chunked = True
self.headers.append((name, value))
else:
if not isinstance(value, types.StringTypes):
value = str(value)
self.headers.append((name, value))
self.set_body(body, content_type=content_type,
content_length=content_length, chunked=chunked)
self.ua = ua
self.chunked = chunked
self.host_hdr = host
self.accept_encoding = accept_encoding
# Finally do the request
return self.do_send()
def _req_headers(self):
# by default all connections are HTTP/1.1
if self.version == (1,1):
httpver = "HTTP/1.1"
else:
httpver = "HTTP/1.0"
# request path
path = self.uri.path or "/"
req_path = urlparse.urlunparse(('','', path, '',
self.uri.query, self.uri.fragment))
# build final request headers
req_headers = [
"%s %s %s\r\n" % (self.method, req_path, httpver),
"Host: %s\r\n" % self.host_hdr,
"User-Agent: %s\r\n" % self.ua,
"Accept-Encoding: %s\r\n" % self.accept_encoding
]
req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers])
req_headers.append('\r\n')
return req_headers
def do_send(self):
tries = 2
while True:
try:
# get socket
self._sock = self.make_connection()
# apply on request filters
self.filters.apply("on_request", self)
# build request headers
self.req_headers = req_headers = self._req_headers()
# send request
log.info('Start request: %s %s', self.method, self.url)
log.debug("Request headers: [%s]", req_headers)
self._sock.sendall("".join(req_headers))
if self.body is not None:
if hasattr(self.body, 'read'):
if hasattr(self.body, 'seek'): self.body.seek(0)
sock.sendfile(self._sock, self.body, self.chunked)
elif isinstance(self.body, types.StringTypes):
sock.send(self._sock, self.body, self.chunked)
else:
sock.sendlines(self._sock, self.body, self.chunked)
if self.chunked: # final chunk
sock.send_chunk(self._sock, "")
return self.start_response()
except socket.gaierror, e:
self.clean_connections()
raise
except socket.error, e:
if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
errno.ECONNREFUSED) or tries <= 0:
self.clean_connections()
raise
if e[0] == errno.EPIPE:
log.debug("Got EPIPE")
self.clean_connections()
except:
class HttpConnection(object):
""" Http Connection object. """
version = (1, 1)
response_class = HttpResponse
def __init__(self,
timeout=sock._GLOBAL_DEFAULT_TIMEOUT,
filters=None,
follow_redirect=False,
force_follow_redirect=False,
max_follow_redirect=MAX_FOLLOW_REDIRECTS,
pool_instance=None,
response_class=None,
**ssl_args):
""" HttpConnection constructor
:param timeout: socket timeout
:param filters: list, list of http filters. see the doc of http filters
for more info
:param follow_redirect: boolean, by default is false. If true,
if the HTTP status is 301, 302 or 303 the client will follow
the location.
:param max_follow_redirect: max number of redirection. If max is reached
the RedirectLimit exception is raised.
:param pool_instance: a pool instance inherited from
`restkit.pool.PoolInterface`
:param ssl_args: ssl arguments. See http://docs.python.org/library/ssl.html
for more information.
"""
self._sock = None
self.timeout = timeout
self.headers = []
self.req_headers = []
self.ua = USER_AGENT
self.url = None
self.follow_redirect = follow_redirect
self.nb_redirections = max_follow_redirect
self.force_follow_redirect = force_follow_redirect
self.method = 'GET'
self.body = None
self.response_body = StringIO()
self.final_url = None
# build filter lists
self.filters = Filters(filters)
self.ssl_args = ssl_args or {}
if not pool_instance:
self.should_close = True
self.pool = None
else:
self.pool = pool_instance
self.should_close = False
if response_class is not None:
self.response_class = response_class
def make_connection(self):
""" initate a connection if needed or reuse a socket"""
# apply on connect filters
self.filters.apply("on_connect", self)
if self._sock is not None:
return self._sock
addr = (self.host, self.port)
s = None
# if we defined a pool use it
if self.pool is not None:
s = self.pool.get(addr)
if not s:
# pool is empty or we don't use a pool
if self.uri.scheme == "https":
s = sock.connect(addr, True, self.timeout, **self.ssl_args)
else:
s = sock.connect(addr, False, self.timeout)
return s
def clean_connections(self):
sock.close(self._sock)
if hasattr(self.pool, 'clear'):
self.pool.clear_host((self.host, self.port))
def release_connection(self, address, socket):
if not self.pool:
return
self.pool.put(address, self._sock)
def parse_url(self, url):
""" parse url and get host/port"""
self.uri = urlparse.urlparse(url)
if self.uri.scheme not in ('http', 'https'):
raise InvalidUrl("None valid url")
host, port = util.parse_netloc(self.uri)
self.host = host
self.port = port
def set_body(self,
body,
content_type=None,
content_length=None,
chunked=False):
""" set HTTP body and manage form if needed """
if not body:
if content_type is not None:
self.headers.append(('Content-Type', content_type))
if self.method in ('POST', 'PUT'):
self.headers.append(("Content-Length", "0"))
return
# set content lengh if needed
if isinstance(body, dict):
if content_type is not None and \
content_type.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(content_type)
boundary = opts.get('boundary', uuid.uuid4().hex)
body, self.headers = multipart_form_encode(
body, self.headers, boundary)
else:
content_type = "application/x-www-form-urlencoded; charset=utf-8"
body = form_encode(body)
elif hasattr(body, "boundary"):
content_type = "multipart/form-data; boundary=%s" % body.boundary
content_length = body.get_size()
if not content_type:
content_type = 'application/octet-stream'
if hasattr(body, 'name'):
content_type = mimetypes.guess_type(body.name)[0]
if not content_length:
if hasattr(body, 'fileno'):
try:
body.flush()
except IOError:
pass
content_length = str(os.fstat(body.fileno())[6])
elif hasattr(body, 'getvalue'):
try:
content_length = str(len(body.getvalue()))
except AttributeError:
pass
elif isinstance(body, types.StringTypes):
body = util.to_bytestring(body)
content_length = len(body)
if content_length:
self.headers.append(("Content-Length", content_length))
if content_type is not None:
self.headers.append(('Content-Type', content_type))
elif not chunked:
raise RequestError("Can't determine content length and" +
"Transfer-Encoding header is not chunked")
self.body = body
def request(self, url, method='GET', body=None, headers=None):
""" make effective request
:param url: str, url string
:param method: str, by default GET. http verbs
:param body: the body, could be a string, an iterator or a file-like object
:param headers: dict or list of tupple, http headers
"""
self._sock = None
self.url = url
self.final_url = url
self.parse_url(url)
self.method = method.upper()
self.headers = []
# headers are better as list
headers = headers or []
if isinstance(headers, dict):
headers = list(headers.items())
ua = USER_AGENT
content_length = None
accept_encoding = 'identity'
chunked = False
content_type = None
# default host
try:
host = self.uri.netloc.encode('ascii')
except UnicodeEncodeError:
host = self.uri.netloc.encode('idna')
# normalize headers
for name, value in headers:
name = name.title()
if name == "User-Agent":
ua = value
elif name == "Content-Type":
content_type = value
elif name == "Content-Length":
content_length = str(value)
elif name == "Accept-Encoding":
accept_encoding = value
elif name == "Host":
host = value
elif name == "Transfer-Encoding":
if value.lower() == "chunked":
chunked = True
self.headers.append((name, value))
else:
if not isinstance(value, types.StringTypes):
value = str(value)
self.headers.append((name, value))
self.set_body(body,
content_type=content_type,
content_length=content_length,
chunked=chunked)
self.ua = ua
self.chunked = chunked
self.host_hdr = host
self.accept_encoding = accept_encoding
# Finally do the request
return self.do_send()
def _req_headers(self):
# by default all connections are HTTP/1.1
if self.version == (1, 1):
httpver = "HTTP/1.1"
else:
httpver = "HTTP/1.0"
# request path
path = self.uri.path or "/"
req_path = urlparse.urlunparse(
('', '', path, '', self.uri.query, self.uri.fragment))
# build final request headers
req_headers = [
"%s %s %s\r\n" % (self.method, req_path, httpver),
"Host: %s\r\n" % self.host_hdr,
"User-Agent: %s\r\n" % self.ua,
"Accept-Encoding: %s\r\n" % self.accept_encoding
]
req_headers.extend(["%s: %s\r\n" % (k, v) for k, v in self.headers])
req_headers.append('\r\n')
return req_headers
def do_send(self):
tries = 2
while True:
try:
# get socket
self._sock = self.make_connection()
# apply on request filters
self.filters.apply("on_request", self)
# build request headers
self.req_headers = req_headers = self._req_headers()
# send request
log.info('Start request: %s %s', self.method, self.url)
log.debug("Request headers: [%s]", req_headers)
self._sock.sendall("".join(req_headers))
if self.body is not None:
if hasattr(self.body, 'read'):
if hasattr(self.body, 'seek'): self.body.seek(0)
sock.sendfile(self._sock, self.body, self.chunked)
elif isinstance(self.body, types.StringTypes):
sock.send(self._sock, self.body, self.chunked)
else:
sock.sendlines(self._sock, self.body, self.chunked)
if self.chunked: # final chunk
sock.send_chunk(self._sock, "")
return self.start_response()
except socket.gaierror, e:
self.clean_connections()
raise
except socket.error, e:
if e[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EPIPE,
errno.ECONNREFUSED) or tries <= 0:
self.clean_connections()
raise
if e[0] == errno.EPIPE:
log.debug("Got EPIPE")
self.clean_connections()
except:
