def __init__(self, name, value, fname=None, filetype=None, filesize=None,
quote=url_quote):
self.quote = quote
self.name = quote(name)
if value is not None and not hasattr(value, 'read'):
value = self.encode_unreadable_value(value)
self.size = len(value)
self.value = value
if fname is not None:
if isinstance(fname, text_type):
fname = fname.encode("utf-8").encode("string_escape").replace('"', '\\"')
else:
fname = fname.encode("string_escape").replace('"', '\\"')
self.fname = fname
if filetype is not None:
filetype = to_bytestring(filetype)
self.filetype = filetype
#if isinstance(value, file) and filesize is None:
if hasattr(value, 'fileno') and filesize is None:
try:
value.flush()
except IOError:
pass
self.size = int(os.fstat(value.fileno())[6])
self._encoded_hdr = None
self._encoded_bdr = None
def __init__(self,
name,
value,
fname=None,
filetype=None,
filesize=None,
quote=url_quote):
self.quote = quote
self.name = quote(name)
if value is not None and not hasattr(value, 'read'):
value = self.encode_unreadable_value(value)
self.size = len(value)
self.value = value
if fname is not None:
if not isinstance(fname, unicode):
fname = fname.decode("utf-8")
self.fname = fname
if filetype is not None:
filetype = to_bytestring(filetype)
self.filetype = filetype
if isinstance(value, file) and filesize is None:
try:
value.flush()
except IOError:
pass
self.size = int(os.fstat(value.fileno())[6])
self._encoded_hdr = None
self._encoded_bdr = None
def sign(self, request, consumer, token):
"""Builds the base signature string."""
key, raw = self.signing_base(request, consumer, token)
hashed = hmac.new(to_bytestring(key), raw, sha)
# Calculate the digest base 64.
return binascii.b2a_base64(hashed.digest())[:-1]
def sign(self, request, consumer, token):
"""Builds the base signature string."""
key, raw = self.signing_base(request, consumer, token)
hashed = hmac.new(to_bytestring(key), raw, sha)
# Calculate the digest base 64.
return binascii.b2a_base64(hashed.digest())[:-1]
def _set_body(self, body):
ctype = self.headers.ipop('content-type', None)
clen = self.headers.ipop('content-length', None)
if isinstance(body, dict):
if ctype is not None and \
ctype.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(ctype)
boundary = opts.get('boundary', uuid.uuid4().hex)
self._body, self.headers = multipart_form_encode(body,
self.headers, boundary)
# at this point content-type is "multipart/form-data"
# we need to set the content type according to the
# correct boundary like
# "multipart/form-data; boundary=%s" % boundary
ctype = self.headers.ipop('content-type', None)
else:
ctype = "application/x-www-form-urlencoded; charset=utf-8"
self._body = form_encode(body)
elif hasattr(body, "boundary") and hasattr(body, "get_size"):
ctype = "multipart/form-data; boundary=%s" % body.boundary
clen = body.get_size()
self._body = body
else:
self._body = body
if not ctype:
ctype = 'application/octet-stream'
if hasattr(self.body, 'name'):
ctype = mimetypes.guess_type(body.name)[0]
if not clen:
if hasattr(self._body, 'fileno'):
try:
self._body.flush()
except IOError:
pass
try:
fno = self._body.fileno()
clen = str(os.fstat(fno)[6])
except IOError:
if not self.is_chunked():
clen = len(self._body.read())
elif hasattr(self._body, 'getvalue') and not \
self.is_chunked():
clen = len(self._body.getvalue())
elif isinstance(self._body, types.StringTypes):
self._body = to_bytestring(self._body)
clen = len(self._body)
if clen is not None:
self.headers['Content-Length'] = clen
# TODO: maybe it's more relevant
# to check if Content-Type is already set in self.headers
# before overiding it
if ctype is not None:
self.headers['Content-Type'] = ctype
def _set_body(self, body):
ctype = self.headers.ipop('content-type', None)
clen = self.headers.ipop('content-length', None)
if isinstance(body, dict):
if ctype is not None and \
ctype.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(ctype)
boundary = opts.get('boundary', uuid.uuid4().hex)
self._body, self.headers = multipart_form_encode(body,
self.headers, boundary)
# at this point content-type is "multipart/form-data"
# we need to set the content type according to the
# correct boundary like
# "multipart/form-data; boundary=%s" % boundary
ctype = self.headers.ipop('content-type', None)
else:
ctype = "application/x-www-form-urlencoded; charset=utf-8"
self._body = form_encode(body)
elif hasattr(body, "boundary") and hasattr(body, "get_size"):
ctype = "multipart/form-data; boundary=%s" % body.boundary
clen = body.get_size()
self._body = body
else:
self._body = body
if not ctype:
ctype = 'application/octet-stream'
if hasattr(self.body, 'name'):
ctype = mimetypes.guess_type(body.name)[0]
if not clen:
if hasattr(self._body, 'fileno'):
try:
self._body.flush()
except IOError:
pass
try:
fno = self._body.fileno()
clen = str(os.fstat(fno)[6])
except IOError:
if not self.is_chunked():
clen = len(self._body.read())
elif hasattr(self._body, 'getvalue') and not \
self.is_chunked():
clen = len(self._body.getvalue())
elif isinstance(self._body, types.StringTypes):
self._body = to_bytestring(self._body)
clen = len(self._body)
if clen is not None:
self.headers['Content-Length'] = clen
# TODO: maybe it's more relevant
# to check if Content-Type is already set in self.headers
# before overiding it
if ctype is not None:
self.headers['Content-Type'] = ctype
def set_body(self, body, headers, chunked=False):
""" set HTTP body and manage form if needed """
content_type = headers.get('CONTENT-TYPE')
content_length = headers.get('CONTENT-LENGTH')
if not body:
if content_type is not None:
self.bheaders.append(('Content-Type', content_type))
if self.method in ('POST', 'PUT'):
self.bheaders.append(("Content-Length", "0"))
return
# set content lengh if needed
if isinstance(body, dict):
if content_type is not None and \
content_type.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(content_type)
boundary = opts.get('boundary', uuid.uuid4().hex)
body, self.bheaders = multipart_form_encode(body,
self.headers, boundary)
else:
content_type = "application/x-www-form-urlencoded; charset=utf-8"
body = form_encode(body)
elif hasattr(body, "boundary"):
content_type = "multipart/form-data; boundary=%s" % body.boundary
content_length = body.get_size()
if not content_type:
content_type = 'application/octet-stream'
if hasattr(body, 'name'):
content_type = mimetypes.guess_type(body.name)[0]
if not content_length:
if hasattr(body, 'fileno'):
try:
body.flush()
except IOError:
pass
content_length = str(os.fstat(body.fileno())[6])
elif hasattr(body, 'getvalue'):
try:
content_length = str(len(body.getvalue()))
except AttributeError:
pass
elif isinstance(body, types.StringTypes):
body = util.to_bytestring(body)
content_length = len(body)
if content_length:
self.bheaders.append(("Content-Length", content_length))
if content_type is not None:
self.bheaders.append(('Content-Type', content_type))
elif not chunked:
raise RequestError("Can't determine content length and " +
"Transfer-Encoding header is not chunked")
self.body = body
def set_body(self, body, headers, chunked=False):
""" set HTTP body and manage form if needed """
content_type = headers.get('CONTENT-TYPE')
content_length = headers.get('CONTENT-LENGTH')
if not body:
if content_type is not None:
self.headers.append(('Content-Type', content_type))
if self.method in ('POST', 'PUT'):
self.headers.append(("Content-Length", "0"))
return
# set content lengh if needed
if isinstance(body, dict):
if content_type is not None and \
content_type.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(content_type)
boundary = opts.get('boundary', uuid.uuid4().hex)
body, self.headers = multipart_form_encode(body,
self.headers, boundary)
else:
content_type = "application/x-www-form-urlencoded; charset=utf-8"
body = form_encode(body)
elif hasattr(body, "boundary"):
content_type = "multipart/form-data; boundary=%s" % body.boundary
content_length = body.get_size()
if not content_type:
content_type = 'application/octet-stream'
if hasattr(body, 'name'):
content_type = mimetypes.guess_type(body.name)[0]
if not content_length:
if hasattr(body, 'fileno'):
try:
body.flush()
except IOError:
pass
content_length = str(os.fstat(body.fileno())[6])
elif hasattr(body, 'getvalue'):
try:
content_length = str(len(body.getvalue()))
except AttributeError:
pass
elif isinstance(body, types.StringTypes):
body = util.to_bytestring(body)
content_length = len(body)
if content_length:
self.headers.append(("Content-Length", content_length))
if content_type is not None:
self.headers.append(('Content-Type', content_type))
elif not chunked:
raise RequestError("Can't determine content length and " +
"Transfer-Encoding header is not chunked")
self.body = body
def _respond(self, http_code, extra_headers, body):
self.send_response(http_code)
keys = []
for key, val in extra_headers:
self.send_header(key, val)
keys.append(key)
if body:
body = to_bytestring(body)
self.end_headers()
self.wfile.write(body)
self.wfile.close()
def _respond(self, http_code, extra_headers, body):
self.send_response(http_code)
keys = []
for k, v in extra_headers:
self.send_header(k, v)
keys.append(k)
if body:
body = to_bytestring(body)
#if body and "Content-Length" not in keys:
# self.send_header("Content-Length", len(body))
self.end_headers()
self.wfile.write(body)
def form_encode(obj, charset="utf8"):
if hasattr( obj, 'items' ):
obj = obj.items()
lines = [
( u"%s=%s" %
( url_quote(key), url_quote(value) )
).encode( charset ) for
(key, value) in obj
]
return to_bytestring( "&".join( lines ) )
def _respond(self, http_code, extra_headers, body):
self.send_response(http_code)
keys = []
for k, v in extra_headers:
self.send_header(k, v)
keys.append(k)
if body:
body = to_bytestring(body)
#if body and "Content-Length" not in keys:
# self.send_header("Content-Length", len(body))
self.end_headers()
self.wfile.write(body)
self.wfile.close()
def _set_body(self, body):
ctype = self.headers.ipop('content-type', None)
clen = self.headers.ipop('content-length', None)
if isinstance(body, dict):
if ctype is not None and \
ctype.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(ctype)
boundary = opts.get('boundary', uuid.uuid4().hex)
self._body, self.headers = multipart_form_encode(
body, self.headers, boundary)
else:
ctype = "application/x-www-form-urlencoded; charset=utf-8"
self._body = form_encode(body)
elif hasattr(body, "boundary"):
ctype = "multipart/form-data; boundary=%s" % self.body.boundary
clen = body.get_size()
self._body = body
else:
self._body = body
if not ctype:
ctype = 'application/octet-stream'
if hasattr(self.body, 'name'):
ctype = mimetypes.guess_type(body.name)[0]
if not clen:
if hasattr(self._body, 'fileno'):
try:
self._body.flush()
except IOError:
pass
try:
fno = self._body.fileno()
clen = str(os.fstat(fno)[6])
except IOError:
if not self.is_chunked():
clen = len(self._body.read())
elif hasattr(self._body, 'getvalue') and not \
self.is_chunked():
clen = len(self._body.getvalue())
elif isinstance(self._body, types.StringTypes):
self._body = to_bytestring(self._body)
clen = len(self._body)
if clen is not None:
self.headers['Content-Length'] = clen
if ctype is not None:
self.headers['Content-Type'] = ctype
def _set_body(self, body):
ctype = self.headers.ipop('content-type', None)
clen = self.headers.ipop('content-length', None)
if isinstance(body, dict):
if ctype is not None and \
ctype.startswith("multipart/form-data"):
type_, opts = cgi.parse_header(ctype)
boundary = opts.get('boundary', uuid.uuid4().hex)
self._body, self.headers = multipart_form_encode(body,
self.headers, boundary)
else:
ctype = "application/x-www-form-urlencoded; charset=utf-8"
self._body = form_encode(body)
elif hasattr(body, "boundary"):
ctype = "multipart/form-data; boundary=%s" % self.body.boundary
clen = body.get_size()
self._body = body
else:
self._body = body
if not ctype:
ctype = 'application/octet-stream'
if hasattr(self.body, 'name'):
ctype = mimetypes.guess_type(body.name)[0]
if not clen:
if hasattr(self._body, 'fileno'):
try:
self._body.flush()
except IOError:
pass
try:
fno = self._body.fileno()
clen = str(os.fstat(fno)[6])
except IOError:
if not self.is_chunked():
clen = len(self._body.read())
elif hasattr(self._body, 'getvalue') and not \
self.is_chunked():
clen = len(self._body.getvalue())
elif isinstance(self._body, types.StringTypes):
self._body = to_bytestring(self._body)
clen = len(self._body)
if clen is not None:
self.headers['Content-Length'] = clen
if ctype is not None:
self.headers['Content-Type'] = ctype
def signing_base(self, request, consumer, token):
if request.normalized_url is None:
raise ValueError("Base URL for request is not set.")
sig = (
escape(request.method),
escape(request.normalized_url),
escape(request.get_normalized_parameters()),
)
key = '%s&' % escape(consumer.secret)
if token:
key += escape(token.secret)
raw = '&'.join(sig)
return to_bytestring(key), raw
def signing_base(self, request, consumer, token):
if not hasattr(request, 'normalized_url') or request.normalized_url is None:
raise ValueError("Base URL for request is not set.")
sig = (
escape(request.method),
escape(request.normalized_url),
escape(request.get_normalized_parameters()),
)
key = '%s&' % escape(consumer.secret)
if token:
key += escape(token.secret)
raw = '&'.join(sig)
return to_bytestring(key), raw
def __init__(self, name, value, fname=None, filetype=None, filesize=None):
self.name = url_quote(name)
if value is not None and not hasattr(value, 'read'):
value = url_quote(value)
self.size = len(value)
self.value = value
if fname is not None:
if isinstance(fname, unicode):
fname = fname.encode("utf-8").encode("string_escape").replace('"', '\\"')
else:
fname = fname.encode("string_escape").replace('"', '\\"')
self.fname = fname
if filetype is not None:
filetype = to_bytestring(filetype)
self.filetype = filetype
if isinstance(value, file) and filesize is None:
try:
value.flush()
except IOError:
pass
self.size = int(os.fstat(value.fileno())[6])
def _host__get(self):
h = to_bytestring(self.parsed_url.netloc)
hdr_host = self.headers.iget("host")
if not hdr_host:
return h
return hdr_host
def form_encode(obj, charset="utf8"):
encoded = url_encode(obj, charset=charset)
return to_bytestring(encoded)
def form_encode(obj, charser="utf8"):
tmp = []
for key, value in obj.items():
tmp.append("%s=%s" % (url_quote(key),
url_quote(value)))
return to_bytestring("&".join(tmp))
def perform(self, request):
""" perform the request. If an error happen it will first try to
restart it """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to perform request: %s %s %s" %
(request.host, request.method, request.path))
tries = 0
while True:
conn = None
try:
# get or create a connection to the remote host
conn = self.get_connection(request)
# send headers
msg = self.make_headers_string(request, conn.extra_headers)
# send body
if request.body is not None:
chunked = request.is_chunked()
if request.headers.iget('content-length') is None and \
not chunked:
raise RequestError(
"Can't determine content length and " +
"Transfer-Encoding header is not chunked")
# handle 100-Continue status
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
hdr_expect = request.headers.iget("expect")
if hdr_expect is not None and \
hdr_expect.lower() == "100-continue":
conn.send(msg)
msg = None
p = HttpStream(SocketReader(conn.socket()),
kind=1,
decompress=True)
if p.status_code != 100:
self.reset_request()
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
return self.response_class(conn, request, p)
chunked = request.is_chunked()
if log.isEnabledFor(logging.DEBUG):
log.debug("send body (chunked: %s)" % chunked)
if isinstance(request.body, (str, )):
if msg is not None:
conn.send(msg + to_bytestring(request.body),
chunked)
else:
conn.send(to_bytestring(request.body), chunked)
else:
if msg is not None:
conn.send(msg)
if hasattr(request.body, 'read'):
if hasattr(request.body, 'seek'):
request.body.seek(0)
conn.sendfile(request.body, chunked)
else:
conn.sendlines(request.body, chunked)
if chunked:
conn.send_chunk("")
else:
conn.send(msg)
return self.get_response(request, conn)
except socket.gaierror as e:
if conn is not None:
conn.release(True)
raise RequestError(str(e))
except socket.timeout as e:
if conn is not None:
conn.release(True)
raise RequestTimeout(str(e))
except socket.error as e:
if log.isEnabledFor(logging.DEBUG):
log.debug("socket error: %s" % str(e))
if conn is not None:
conn.close()
errors = (errno.EAGAIN, errno.EPIPE, errno.EBADF,
errno.ECONNRESET)
if e[0] not in errors or tries >= self.max_tries:
raise RequestError("socket.error: %s" % str(e))
# should raised an exception in other cases
request.maybe_rewind(msg=str(e))
except NoMoreData as e:
if conn is not None:
conn.release(True)
request.maybe_rewind(msg=str(e))
if tries >= self.max_tries:
raise
except BadStatusLine:
if conn is not None:
conn.release(True)
# should raised an exception in other cases
request.maybe_rewind(msg="bad status line")
if tries >= self.max_tries:
raise
except Exception:
# unkown error
log.debug("unhandled exception %s" % traceback.format_exc())
if conn is not None:
conn.release(True)
raise
tries += 1
self._pool.backend_mod.sleep(self.wait_tries)
def _host__get(self):
h = to_bytestring(self.parsed_url.netloc)
hdr_host = self.headers.iget("host")
if not hdr_host:
return h
return hdr_host
def form_encode(obj, charset="utf8"):
encoded = url_encode(obj, charset=charset)
return to_bytestring(encoded)
def request(self, url, method='GET', body=None, headers=None):
""" make effective request
:param url: str, url string
:param method: str, by default GET. http verbs
:param body: the body, could be a string, an iterator or a file-like object
:param headers: dict or list of tupple, http headers
"""
self.parser = Parser.parse_response(should_close=self.should_close)
self._sock = None
self.url = url
self.final_url = url
self.parse_url(url)
self.method = method.upper()
# headers are better as list
headers = headers or []
if isinstance(headers, dict):
headers = list(headers.items())
ua = USER_AGENT
normalized_headers = []
content_len = None
accept_encoding = 'identity'
chunked = False
# default host
try:
host = self.uri.netloc.encode('ascii')
except UnicodeEncodeError:
host = self.uri.netloc.encode('idna')
# normalize headers
for name, value in headers:
name = name.title()
if name == "User-Agent":
ua = value
elif name == "Content-Length":
content_len = str(value)
elif name == "Accept-Encoding":
accept_encoding = 'identity'
elif name == "Host":
host = value
elif name == "Transfer-Encoding":
if value.lower() == "chunked":
chunked = True
normalized_headers.append((name, value))
else:
if not isinstance(value, types.StringTypes):
value = str(value)
normalized_headers.append((name, value))
# set content lengh if needed
if body and body is not None:
if not content_len:
if hasattr(body, 'fileno'):
try:
body.flush()
except IOError:
pass
content_len = str(os.fstat(body.fileno())[6])
elif hasattr(body, 'getvalue'):
try:
content_len = str(len(body.getvalue()))
except AttributeError:
pass
elif isinstance(body, types.StringTypes):
body = util.to_bytestring(body)
content_len = len(body)
if content_len:
normalized_headers.append(("Content-Length", content_len))
elif not chunked:
raise RequestError("Can't determine content length and" +
"Transfer-Encoding header is not chunked")
if self.method in ('POST', 'PUT') and not body:
normalized_headers.append(("Content-Length", "0"))
self.body = body
self.headers = normalized_headers
self.ua = ua
self.chunked = chunked
self.host_hdr = host
self.accept_encoding = accept_encoding
# Finally do the request
return self.do_send()
def perform(self, request):
""" perform the request. If an error happen it will first try to
restart it """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to perform request: %s %s %s" %
(request.host, request.method, request.path))
tries = 0
while True:
conn = None
try:
# get or create a connection to the remote host
conn = self.get_connection(request)
# send headers
msg = self.make_headers_string(request, conn.extra_headers)
# send body
if request.body is not None:
chunked = request.is_chunked()
if request.headers.iget('content-length') is None and \
not chunked:
raise RequestError(
"Can't determine content length and " +
"Transfer-Encoding header is not chunked")
# handle 100-Continue status
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
hdr_expect = request.headers.iget("expect")
if hdr_expect is not None and \
hdr_expect.lower() == "100-continue":
conn.send(msg)
msg = None
p = HttpStream(SocketReader(conn.socket()),
kind=1,
decompress=True)
if p.status_code != 100:
self.reset_request()
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
return self.response_class(conn, request, p)
chunked = request.is_chunked()
if log.isEnabledFor(logging.DEBUG):
log.debug("send body (chunked: %s)" % chunked)
if isinstance(request.body, types.StringTypes):
if msg is not None:
conn.send(msg + to_bytestring(request.body),
chunked)
else:
conn.send(to_bytestring(request.body), chunked)
else:
if msg is not None:
conn.send(msg)
if hasattr(request.body, 'read'):
if hasattr(request.body, 'seek'):
request.body.seek(0)
conn.sendfile(request.body, chunked)
else:
conn.sendlines(request.body, chunked)
if chunked:
conn.send_chunk("")
else:
conn.send(msg)
return self.get_response(request, conn)
except socket.gaierror, e:
if conn is not None:
conn.release(True)
raise RequestError(str(e))
except socket.timeout, e:
if conn is not None:
conn.release(True)
raise RequestTimeout(str(e))
def perform(self, request):
""" perform the request. If an error happen it will first try to
restart it """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to perform request: %s %s %s" %
(request.host, request.method, request.path))
tries = 0
while True:
conn = None
try:
# get or create a connection to the remote host
conn = self.get_connection(request)
# send headers
msg = self.make_headers_string(request,
conn.extra_headers)
# send body
if request.body is not None:
chunked = request.is_chunked()
if request.headers.iget('content-length') is None and \
not chunked:
raise RequestError(
"Can't determine content length and " +
"Transfer-Encoding header is not chunked")
# handle 100-Continue status
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
hdr_expect = request.headers.iget("expect")
if hdr_expect is not None and \
hdr_expect.lower() == "100-continue":
conn.send(msg)
msg = None
p = HttpStream(SocketReader(conn.socket()), kind=1,
decompress=True)
if p.status_code != 100:
self.reset_request()
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
return self.response_class(conn, request, p)
chunked = request.is_chunked()
if log.isEnabledFor(logging.DEBUG):
log.debug("send body (chunked: %s)" % chunked)
if isinstance(request.body, types.StringTypes):
if msg is not None:
conn.send(msg + to_bytestring(request.body),
chunked)
else:
conn.send(to_bytestring(request.body), chunked)
else:
if msg is not None:
conn.send(msg)
if hasattr(request.body, 'read'):
if hasattr(request.body, 'seek'):
request.body.seek(0)
conn.sendfile(request.body, chunked)
else:
conn.sendlines(request.body, chunked)
if chunked:
conn.send_chunk("")
else:
conn.send(msg)
return self.get_response(request, conn)
except socket.gaierror, e:
if conn is not None:
conn.release(True)
raise RequestError(str(e))
except socket.timeout, e:
if conn is not None:
conn.release(True)
raise RequestTimeout(str(e))