def validate_etags(request, response, autotags=False):
"""Validate the current ETag against If-Match, If-None-Match headers.
If autotags is True, an ETag response-header value will be provided
from an MD5 hash of the response body (unless some other code has
already provided an ETag header). If False (the default), the ETag
will not be automatic.
WARNING: the autotags feature is not designed for URL's which allow
methods other than GET. For example, if a POST to the same URL returns
no content, the automatic ETag will be incorrect, breaking a fundamental
use for entity tags in a possibly destructive fashion. Likewise, if you
raise 304 Not Modified, the response body will be empty, the ETag hash
will be incorrect, and your application will break.
See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.24
"""
# Guard against being run twice.
if hasattr(response, "ETag"):
return
status, reason, msg = valid_status(response.status)
etag = response.headers.get('ETag')
# Automatic ETag generation. See warning in docstring.
if (not etag) and autotags:
if status == 200:
etag = response.collapse_body()
etag = '"%s"' % hashlib.md5.new(etag).hexdigest()
response.headers['ETag'] = etag
response.ETag = etag
# "If the request would, without the If-Match header field, result in
# anything other than a 2xx or 412 status, then the If-Match header
# MUST be ignored."
if status >= 200 and status <= 299:
conditions = request.headers.elements('If-Match') or []
conditions = [str(x) for x in conditions]
if conditions and not (conditions == ["*"] or etag in conditions):
return HTTPError(
request, response, 412,
"If-Match failed: ETag %r did not match %r" %
(etag, conditions))
conditions = request.headers.elements('If-None-Match') or []
conditions = [str(x) for x in conditions]
if conditions == ["*"] or etag in conditions:
if request.method in ("GET", "HEAD"):
return Redirect(request, response, [], 304)
else:
return HTTPError(
request, response, 412,
"If-None-Match failed: ETag %r matched %r" %
(etag, conditions))
def __call__(self, environ, start_response, exc_info=None):
request, response = self.getRequestResponse(environ)
try:
req = Request(request, response)
v = self.send(req, "request", self.channel, errors=True)
if v:
if issubclass(type(v), basestring):
response.body = v
res = Response(response)
self.send(res, "response", self.channel)
elif isinstance(v, HTTPError):
self._handleError(v)
elif isinstance(v, wrappers.Response):
res = Response(v)
self.send(res, "response", self.channel)
else:
assert v, "type(v) == %s" % type(v)
else:
error = NotFound(request, response)
self._handleError(error)
except:
error = HTTPError(request, response, 500, error=format_exc())
self._handleError(error)
finally:
body = response.process()
start_response(response.status, response.headers.items(), exc_info)
return [body]
def request_success_or_filtered(self, evt, handler, retval):
if retval:
request, response = evt.args[:2]
request.handled = True
if isinstance(retval, HTTPError):
self.push(retval, "httperror", self.channel)
elif isinstance(retval, wrappers.Response):
self.push(Response(retval))
elif isinstance(retval, Value):
if retval.result and not retval.errors:
response.body = retval.value
self.push(Response(response))
elif retval.errors:
message = "Request Failed"
error = retval.value
self.push(HTTPError(request, response, 500, message,
error))
else:
if retval.manager is None:
retval.manager = self
retval.event = evt
retval.onSet = "valuechanged", self
elif type(retval) is not bool:
response.body = retval
self.push(Response(response))
def request(method, url, payload=None):
if type(api_key) != StringType:
raise Exception("API key '%s' is not a string" % (api_key, ))
uri = "%s%s" % (root, url)
headers = {"Authorization": "Bearer %s" % (api_key, )}
if payload:
headers["Content-Type"] = "application/json; charset=utf-8"
body = json.dumps(payload)
else:
body = None
(response, output) = http.request(uri, method, body, headers)
results = None
if 'content-type' in response:
contentType = str.split(response['content-type'], ';', 1)
if contentType[0] == "application/json":
results = json.loads(output)
if response.status != 200:
if results:
message = results["message"]
else:
message = output
raise HTTPError(response.status, message)
return (results, response)
def check_auth(request, response, realm, users, encrypt=None):
"""Check Authentication
If an authorization header contains credentials, return True, else False.
@param realm: The authentication realm.
@type realm: str
@param users: A dict of the form: {username: password} or a callable
returning a dict.
@type users: dict or callable
@param encrypt: Callable used to encrypt the password returned from
the user-agent. if None it defaults to a md5 encryption.
@type encrypt: callable
"""
if 'authorization' in request.headers:
# make sure the provided credentials are correctly set
ah = _httpauth.parseAuthorization(request.headers['authorization'])
if ah is None:
return HTTPError(request, response, 400)
if not encrypt:
encrypt = _httpauth.DIGEST_AUTH_ENCODERS[_httpauth.MD5]
if callable(users):
try:
# backward compatibility
users = users() # expect it to return a dictionary
if not isinstance(users, dict):
raise ValueError, "Authentication users must be a dict"
# fetch the user password
password = users.get(ah["username"], None)
except TypeError:
# returns a password (encrypted or clear text)
password = users(ah["username"])
else:
if not isinstance(users, dict):
raise ValueError, "Authentication users must be a dict"
# fetch the user password
password = users.get(ah["username"], None)
# validate the authorization by re-computing it here
# and compare it with what the user-agent provided
if _httpauth.checkResponse(ah,
password,
method=request.method,
encrypt=encrypt,
realm=realm):
request.login = ah["username"]
return True
request.login = False
return False
def valuechanged(self, value):
request, response = value.event.args[:2]
if value.result and not value.errors:
response.body = value.value
self.push(Response(response))
else:
# This possibly never occurs.
message = "Request Failed"
error = value.value
self.push(HTTPError(request, response, 500, message, error))
def gzip(response, level=1, mime_types=['text/html', 'text/plain']):
"""Try to gzip the response body if Content-Type in mime_types.
response.headers['Content-Type'] must be set to one of the
values in the mime_types arg before calling this function.
No compression is performed if any of the following hold:
* The client sends no Accept-Encoding request header
* No 'gzip' or 'x-gzip' is present in the Accept-Encoding header
* No 'gzip' or 'x-gzip' with a qvalue > 0 is present
* The 'identity' value is given with a qvalue > 0.
"""
if not response.body:
# Response body is empty (might be a 304 for instance)
return response
# If returning cached content (which should already have been gzipped),
# don't re-zip.
if getattr(response.request, "cached", False):
return response
acceptable = response.request.headers.elements('Accept-Encoding')
if not acceptable:
# If no Accept-Encoding field is present in a request,
# the server MAY assume that the client will accept any
# content coding. In this case, if "identity" is one of
# the available content-codings, then the server SHOULD use
# the "identity" content-coding, unless it has additional
# information that a different content-coding is meaningful
# to the client.
return response
ct = response.headers.get('Content-Type', 'text/html').split(';')[0]
for coding in acceptable:
if coding.value == 'identity' and coding.qvalue != 0:
return response
if coding.value in ('gzip', 'x-gzip'):
if coding.qvalue == 0:
return response
if ct in mime_types:
# Return a generator that compresses the page
varies = response.headers.get("Vary", "")
varies = [x.strip() for x in varies.split(",") if x.strip()]
if "Accept-Encoding" not in varies:
varies.append("Accept-Encoding")
response.headers['Vary'] = ", ".join(varies)
response.headers['Content-Encoding'] = 'gzip'
response.body = compress(response.body, level)
if response.headers.has_key("Content-Length"):
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
return response
return HTTPError(response.request, response, 406, "identity, gzip")
def validate_since(request, response):
"""Validate the current Last-Modified against If-Modified-Since headers.
If no code has set the Last-Modified response header, then no validation
will be performed.
"""
lastmod = response.headers.get('Last-Modified')
if lastmod:
status, reason, msg = valid_status(response.status)
since = request.headers.get('If-Unmodified-Since')
if since and since != lastmod:
if (status >= 200 and status <= 299) or status == 412:
return HTTPError(request, response, 412)
since = request.headers.get('If-Modified-Since')
if since and since == lastmod:
if (status >= 200 and status <= 299) or status == 304:
if request.method in ("GET", "HEAD"):
return Redirect(request, response, [], 304)
else:
return HTTPError(request, response, 412)
def _parseBody(self, request, response, params):
body = request.body
headers = request.headers
if "Content-Type" not in headers:
headers["Content-Type"] = ""
try:
form = FieldStorage(fp=body,
headers=headers,
environ={"REQUEST_METHOD": "POST"},
keep_blank_values=True)
except Exception, e:
if e.__class__.__name__ == 'MaxSizeExceeded':
# Post data is too big
return HTTPError(request, response, 413)
else:
raise
def request(self, event, request, response):
if self.path is not None and not request.path.startswith(self.path):
return
req = event
path = request.path
if self.path is not None:
path = path[len(self.path):]
req.path = path
self._request = request
self._response = response
try:
return "".join(self.app(self._createEnviron(),
self.start_response))
except Exception, error:
status = error.code
message = error.message
error = _exc_info()
return HTTPError(request, response, status, message, error)
def _on_request(self, event, request, response):
if self.path and not request.path.startswith(self.path):
return
req = event
path = request.path
if self.path is not None:
path = path[len(self.path):]
req.path = path
self._request = request
self._response = response
try:
return self.app(self.createEnviron(), self.start_response)
except Exception, error:
status = 500
message = str(error)
error = _exc_info()
etype, evalue, etraceback = _exc_info()
error = (etype, evalue, format_tb(etraceback))
return HTTPError(request, response, status, message, error)
def get(self, url):
try:
u = urllib2.urlopen(url)
except urllib2.HTTPError, e:
raise HTTPError(e.code, e)
def get(self, url):
headers, response = self.http.request(url)
if headers['status'] != '200':
raise HTTPError(int(headers['status']), headers)
return headers, response
def request_failure(self, evt, handler, error):
request, response = evt.args[:2]
message = "Request Failed"
self._handleError(HTTPError(request, response, 500, message, error))
def read(self, sock, data):
"""Read Event Handler
Process any incoming data appending it to an internal buffer.
Split the buffer by the standard HTTP delimiter CRLF and create
Raw Event per line. Any unfinished lines of text, leave in the buffer.
"""
if sock in self._clients:
request, response = self._clients[sock]
if response.done:
del self._clients[sock]
if sock in self._clients:
request, response = self._clients[sock]
request.body.write(data)
contentLength = int(request.headers.get("Content-Length", "0"))
if not request.body.tell() == contentLength:
return
else:
requestline, data = data.split("\n", 1)
requestline = requestline.strip()
method, path, protocol = requestline.split(" ", 2)
scheme, location, path, params, qs, frag = urlparse(path)
protocol = tuple(map(int, protocol[5:].split(".")))
request = wrappers.Request(sock, method, scheme, path, protocol,
qs)
response = wrappers.Response(sock, request)
self._clients[sock] = request, response
if frag:
error = HTTPError(request, response, 400)
return self.push(error, "httperror", self.channel)
if params:
path = "%s;%s" % (path, params)
# Unquote the path+params (e.g. "/this%20path" -> "this path").
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
#
# But note that "...a URI must be separated into its components
# before the escaped characters within those components can be
# safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2
path = "%2F".join(map(unquote, quoted_slash.split(path)))
# Compare request and server HTTP protocol versions, in case our
# server does not support the requested protocol. Limit our output
# to min(req, server). We want the following output:
# request server actual written supported response
# protocol protocol response protocol feature set
# a 1.0 1.0 1.0 1.0
# b 1.0 1.1 1.1 1.0
# c 1.1 1.0 1.0 1.0
# d 1.1 1.1 1.1 1.1
# Notice that, in (b), the response will be "HTTP/1.1" even though
# the client only understands 1.0. RFC 2616 10.5.6 says we should
# only return 505 if the _major_ version is different.
if not request.protocol[0] == request.server_protocol[0]:
error = HTTPError(request, response, 505)
return self.push(error, "httperror", self.channel)
rp = request.protocol
sp = request.server_protocol
response.protocol = "HTTP/%s.%s" % min(rp, sp)
headers, body = parseHeaders(StringIO(data))
request.headers = headers
request.body.write(body)
if headers.get("Expect", "") == "100-continue":
return self.simple(sock, 100)
contentLength = int(headers.get("Content-Length", "0"))
if not request.body.tell() == contentLength:
return
# Persistent connection support
if request.protocol == (1, 1):
# Both server and client are HTTP/1.1
if request.headers.get("Connection", "").lower() == "close":
response.close = True
else:
# Either the server or client (or both) are HTTP/1.0
if request.headers.get("Connection", "").lower() != "keep-alive":
response.close = True
request.body.seek(0)
self.push(Request(request, response), "request", "web")
def _fetch(self, url):
"""Fetch results from JSON endpoint and return as dict."""
f = urlfetch.fetch(url)
if f.status_code != 200:
raise HTTPError(f.status_code, f.headers)
return json.loads(f.content)
def serve_file(request,
response,
path,
type=None,
disposition=None,
name=None):
"""Set status, headers, and body in order to serve the given file.
The Content-Type header will be set to the type arg, if provided.
If not provided, the Content-Type will be guessed by the file extension
of the 'path' argument.
If disposition is not None, the Content-Disposition header will be set
to "<disposition>; filename=<name>". If name is None, it will be set
to the basename of path. If disposition is None, no Content-Disposition
header will be written.
"""
if not os.path.isabs(path):
raise ValueError("'%s' is not an absolute path." % path)
try:
st = os.stat(path)
except OSError:
return NotFound(request, response)
# Check if path is a directory.
if stat.S_ISDIR(st.st_mode):
# Let the caller deal with it as they like.
return NotFound(request, response)
# Set the Last-Modified response header, so that
# modified-since validation code can work.
response.headers['Last-Modified'] = formatdate(st.st_mtime)
validate_since(request, response)
if type is None:
# Set content-type based on filename extension
ext = ""
i = path.rfind('.')
if i != -1:
ext = path[i:].lower()
type = mimetypes.types_map.get(ext, "text/plain")
response.headers['Content-Type'] = type
if disposition is not None:
if name is None:
name = os.path.basename(path)
cd = '%s; filename="%s"' % (disposition, name)
response.headers["Content-Disposition"] = cd
# Set Content-Length and use an iterable (file object)
# this way CP won't load the whole file in memory
c_len = st.st_size
bodyfile = open(path, 'rb')
# HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code
if request.protocol >= (1, 1):
response.headers["Accept-Ranges"] = "bytes"
r = get_ranges(request.headers.get('Range'), c_len)
if r == []:
response.headers['Content-Range'] = "bytes */%s" % c_len
message = "Invalid Range (first-byte-pos greater than Content-Length)"
return HTTPError(request, response, 416, message)
if r:
if len(r) == 1:
# Return a single-part response.
start, stop = r[0]
r_len = stop - start
response.status = "206 Partial Content"
response.headers['Content-Range'] = ("bytes %s-%s/%s" %
(start, stop - 1, c_len))
response.headers['Content-Length'] = r_len
bodyfile.seek(start)
response.body = bodyfile.read(r_len)
else:
# Return a multipart/byteranges response.
response.status = "206 Partial Content"
boundary = mimetools.choose_boundary()
ct = "multipart/byteranges; boundary=%s" % boundary
response.headers['Content-Type'] = ct
if response.headers.has_key("Content-Length"):
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
def file_ranges():
# Apache compatibility:
yield "\r\n"
for start, stop in r:
yield "--" + boundary
yield "\r\nContent-type: %s" % type
yield ("\r\nContent-range: bytes %s-%s/%s\r\n\r\n" %
(start, stop - 1, c_len))
bodyfile.seek(start)
yield bodyfile.read(stop - start)
yield "\r\n"
# Final boundary
yield "--" + boundary + "--"
# Apache compatibility:
yield "\r\n"
response.body = file_ranges()
else:
response.headers['Content-Length'] = c_len
response.body = bodyfile
else:
response.headers['Content-Length'] = c_len
response.body = bodyfile
return response
class Browser:
"""A class for making calls to lazr.restful web services."""
NOT_MODIFIED = object()
MAX_RETRIES = 6
def __init__(self, service_root, credentials, cache=None, timeout=None,
proxy_info=None, user_agent=None, max_retries=MAX_RETRIES):
"""Initialize, possibly creating a cache.
If no cache is provided, a temporary directory will be used as
a cache. The temporary directory will be automatically removed
when the Python process exits.
"""
if cache is None:
cache = tempfile.mkdtemp()
atexit.register(shutil.rmtree, cache)
if isinstance(cache, basestring):
cache = MultipleRepresentationCache(cache)
self._connection = service_root.httpFactory(
credentials, cache, timeout, proxy_info)
self.user_agent = user_agent
self.max_retries = max_retries
def _request_and_retry(self, url, method, body, headers):
for retry_count in range(0, self.max_retries+1):
response, content = self._connection.request(
url, method=method, body=body, headers=headers)
if (response.status in [502, 503]
and retry_count < self.max_retries):
# The server returned a 502 or 503. Sleep for 0, 1, 2,
# 4, 8, 16, ... seconds and try again.
sleep_for = int(2**(retry_count-1))
sleep(sleep_for)
else:
break
# Either the request succeeded or we gave up.
return response, content
def _request(self, url, data=None, method='GET',
media_type='application/json', extra_headers=None):
"""Create an authenticated request object."""
# If the user is trying to get data that has been redacted,
# give a helpful message.
if url == "tag:launchpad.net:2008:redacted":
raise ValueError("You tried to access a resource that you "
"don't have the server-side permission to see.")
# Add extra headers for the request.
headers = {'Accept' : media_type}
if self.user_agent is not None:
headers['User-Agent'] = self.user_agent
if isinstance(self._connection.cache, MultipleRepresentationCache):
self._connection.cache.request_media_type = media_type
if extra_headers is not None:
headers.update(extra_headers)
# Make the request.
try:
response, content = self._request_and_retry(
str(url), method=method, body=data, headers=headers)
except SSLHandshakeError, e:
msg = str(e)
if "SSL3_GET_SERVER_CERTIFICATE:certificate verify failed" in msg:
raise SSLHandshakeError(
"%s: perhaps set LP_DISABLE_SSL_CERTIFICATE_VALIDATION if "
"certificate validation failed and you genuinely trust the "
"remote server." % (e,))
raise
if response.status == 304:
# The resource didn't change.
if content == '':
if ('If-None-Match' in headers
or 'If-Modified-Since' in headers):
# The caller made a conditional request, and the
# condition failed. Rather than send an empty
# representation, which might be misinterpreted,
# send a special object that will let the calling code know
# that the resource was not modified.
return response, self.NOT_MODIFIED
else:
# The caller didn't make a conditional request,
# but the response code is 304 and there's no
# content. The only way to handle this is to raise
# an error.
#
# We don't use error_for() here because 304 is not
# normally considered an error condition.
raise HTTPError(response, content)
else:
# XXX leonardr 2010/04/12 bug=httplib2#97
#
# Why is this check here? Why would there ever be any
# content when the response code is 304? It's because of
# an httplib2 bug that sometimes sets a 304 response
# code when caching retrieved documents. When the
# cached document is retrieved, we get a 304 response
# code and a full representation.
#
# Since the cache lookup succeeded, the 'real'
# response code is 200. This code undoes the bad
# behavior in httplib2.
response.status = 200
return response, content
# Turn non-2xx responses into appropriate HTTPError subclasses.
error = error_for(response, content)
if error is not None:
raise error
return response, content
