def validate_etags(request, response, autotags=False): """Validate the current ETag against If-Match, If-None-Match headers. If autotags is True, an ETag response-header value will be provided from an MD5 hash of the response body (unless some other code has already provided an ETag header). If False (the default), the ETag will not be automatic. WARNING: the autotags feature is not designed for URL's which allow methods other than GET. For example, if a POST to the same URL returns no content, the automatic ETag will be incorrect, breaking a fundamental use for entity tags in a possibly destructive fashion. Likewise, if you raise 304 Not Modified, the response body will be empty, the ETag hash will be incorrect, and your application will break. See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.24 """ # Guard against being run twice. if hasattr(response, "ETag"): return status, reason, msg = valid_status(response.status) etag = response.headers.get('ETag') # Automatic ETag generation. See warning in docstring. if (not etag) and autotags: if status == 200: etag = response.collapse_body() etag = '"%s"' % hashlib.md5.new(etag).hexdigest() response.headers['ETag'] = etag response.ETag = etag # "If the request would, without the If-Match header field, result in # anything other than a 2xx or 412 status, then the If-Match header # MUST be ignored." if status >= 200 and status <= 299: conditions = request.headers.elements('If-Match') or [] conditions = [str(x) for x in conditions] if conditions and not (conditions == ["*"] or etag in conditions): return HTTPError( request, response, 412, "If-Match failed: ETag %r did not match %r" % (etag, conditions)) conditions = request.headers.elements('If-None-Match') or [] conditions = [str(x) for x in conditions] if conditions == ["*"] or etag in conditions: if request.method in ("GET", "HEAD"): return Redirect(request, response, [], 304) else: return HTTPError( request, response, 412, "If-None-Match failed: ETag %r matched %r" % (etag, conditions))
def __call__(self, environ, start_response, exc_info=None): request, response = self.getRequestResponse(environ) try: req = Request(request, response) v = self.send(req, "request", self.channel, errors=True) if v: if issubclass(type(v), basestring): response.body = v res = Response(response) self.send(res, "response", self.channel) elif isinstance(v, HTTPError): self._handleError(v) elif isinstance(v, wrappers.Response): res = Response(v) self.send(res, "response", self.channel) else: assert v, "type(v) == %s" % type(v) else: error = NotFound(request, response) self._handleError(error) except: error = HTTPError(request, response, 500, error=format_exc()) self._handleError(error) finally: body = response.process() start_response(response.status, response.headers.items(), exc_info) return [body]
def request_success_or_filtered(self, evt, handler, retval): if retval: request, response = evt.args[:2] request.handled = True if isinstance(retval, HTTPError): self.push(retval, "httperror", self.channel) elif isinstance(retval, wrappers.Response): self.push(Response(retval)) elif isinstance(retval, Value): if retval.result and not retval.errors: response.body = retval.value self.push(Response(response)) elif retval.errors: message = "Request Failed" error = retval.value self.push(HTTPError(request, response, 500, message, error)) else: if retval.manager is None: retval.manager = self retval.event = evt retval.onSet = "valuechanged", self elif type(retval) is not bool: response.body = retval self.push(Response(response))
def request(method, url, payload=None): if type(api_key) != StringType: raise Exception("API key '%s' is not a string" % (api_key, )) uri = "%s%s" % (root, url) headers = {"Authorization": "Bearer %s" % (api_key, )} if payload: headers["Content-Type"] = "application/json; charset=utf-8" body = json.dumps(payload) else: body = None (response, output) = http.request(uri, method, body, headers) results = None if 'content-type' in response: contentType = str.split(response['content-type'], ';', 1) if contentType[0] == "application/json": results = json.loads(output) if response.status != 200: if results: message = results["message"] else: message = output raise HTTPError(response.status, message) return (results, response)
def check_auth(request, response, realm, users, encrypt=None): """Check Authentication If an authorization header contains credentials, return True, else False. @param realm: The authentication realm. @type realm: str @param users: A dict of the form: {username: password} or a callable returning a dict. @type users: dict or callable @param encrypt: Callable used to encrypt the password returned from the user-agent. if None it defaults to a md5 encryption. @type encrypt: callable """ if 'authorization' in request.headers: # make sure the provided credentials are correctly set ah = _httpauth.parseAuthorization(request.headers['authorization']) if ah is None: return HTTPError(request, response, 400) if not encrypt: encrypt = _httpauth.DIGEST_AUTH_ENCODERS[_httpauth.MD5] if callable(users): try: # backward compatibility users = users() # expect it to return a dictionary if not isinstance(users, dict): raise ValueError, "Authentication users must be a dict" # fetch the user password password = users.get(ah["username"], None) except TypeError: # returns a password (encrypted or clear text) password = users(ah["username"]) else: if not isinstance(users, dict): raise ValueError, "Authentication users must be a dict" # fetch the user password password = users.get(ah["username"], None) # validate the authorization by re-computing it here # and compare it with what the user-agent provided if _httpauth.checkResponse(ah, password, method=request.method, encrypt=encrypt, realm=realm): request.login = ah["username"] return True request.login = False return False
def valuechanged(self, value): request, response = value.event.args[:2] if value.result and not value.errors: response.body = value.value self.push(Response(response)) else: # This possibly never occurs. message = "Request Failed" error = value.value self.push(HTTPError(request, response, 500, message, error))
def gzip(response, level=1, mime_types=['text/html', 'text/plain']): """Try to gzip the response body if Content-Type in mime_types. response.headers['Content-Type'] must be set to one of the values in the mime_types arg before calling this function. No compression is performed if any of the following hold: * The client sends no Accept-Encoding request header * No 'gzip' or 'x-gzip' is present in the Accept-Encoding header * No 'gzip' or 'x-gzip' with a qvalue > 0 is present * The 'identity' value is given with a qvalue > 0. """ if not response.body: # Response body is empty (might be a 304 for instance) return response # If returning cached content (which should already have been gzipped), # don't re-zip. if getattr(response.request, "cached", False): return response acceptable = response.request.headers.elements('Accept-Encoding') if not acceptable: # If no Accept-Encoding field is present in a request, # the server MAY assume that the client will accept any # content coding. In this case, if "identity" is one of # the available content-codings, then the server SHOULD use # the "identity" content-coding, unless it has additional # information that a different content-coding is meaningful # to the client. return response ct = response.headers.get('Content-Type', 'text/html').split(';')[0] for coding in acceptable: if coding.value == 'identity' and coding.qvalue != 0: return response if coding.value in ('gzip', 'x-gzip'): if coding.qvalue == 0: return response if ct in mime_types: # Return a generator that compresses the page varies = response.headers.get("Vary", "") varies = [x.strip() for x in varies.split(",") if x.strip()] if "Accept-Encoding" not in varies: varies.append("Accept-Encoding") response.headers['Vary'] = ", ".join(varies) response.headers['Content-Encoding'] = 'gzip' response.body = compress(response.body, level) if response.headers.has_key("Content-Length"): # Delete Content-Length header so finalize() recalcs it. del response.headers["Content-Length"] return response return HTTPError(response.request, response, 406, "identity, gzip")
def validate_since(request, response): """Validate the current Last-Modified against If-Modified-Since headers. If no code has set the Last-Modified response header, then no validation will be performed. """ lastmod = response.headers.get('Last-Modified') if lastmod: status, reason, msg = valid_status(response.status) since = request.headers.get('If-Unmodified-Since') if since and since != lastmod: if (status >= 200 and status <= 299) or status == 412: return HTTPError(request, response, 412) since = request.headers.get('If-Modified-Since') if since and since == lastmod: if (status >= 200 and status <= 299) or status == 304: if request.method in ("GET", "HEAD"): return Redirect(request, response, [], 304) else: return HTTPError(request, response, 412)
def _parseBody(self, request, response, params): body = request.body headers = request.headers if "Content-Type" not in headers: headers["Content-Type"] = "" try: form = FieldStorage(fp=body, headers=headers, environ={"REQUEST_METHOD": "POST"}, keep_blank_values=True) except Exception, e: if e.__class__.__name__ == 'MaxSizeExceeded': # Post data is too big return HTTPError(request, response, 413) else: raise
def request(self, event, request, response): if self.path is not None and not request.path.startswith(self.path): return req = event path = request.path if self.path is not None: path = path[len(self.path):] req.path = path self._request = request self._response = response try: return "".join(self.app(self._createEnviron(), self.start_response)) except Exception, error: status = error.code message = error.message error = _exc_info() return HTTPError(request, response, status, message, error)
def _on_request(self, event, request, response): if self.path and not request.path.startswith(self.path): return req = event path = request.path if self.path is not None: path = path[len(self.path):] req.path = path self._request = request self._response = response try: return self.app(self.createEnviron(), self.start_response) except Exception, error: status = 500 message = str(error) error = _exc_info() etype, evalue, etraceback = _exc_info() error = (etype, evalue, format_tb(etraceback)) return HTTPError(request, response, status, message, error)
def get(self, url): try: u = urllib2.urlopen(url) except urllib2.HTTPError, e: raise HTTPError(e.code, e)
def get(self, url): headers, response = self.http.request(url) if headers['status'] != '200': raise HTTPError(int(headers['status']), headers) return headers, response
def request_failure(self, evt, handler, error): request, response = evt.args[:2] message = "Request Failed" self._handleError(HTTPError(request, response, 500, message, error))
def read(self, sock, data): """Read Event Handler Process any incoming data appending it to an internal buffer. Split the buffer by the standard HTTP delimiter CRLF and create Raw Event per line. Any unfinished lines of text, leave in the buffer. """ if sock in self._clients: request, response = self._clients[sock] if response.done: del self._clients[sock] if sock in self._clients: request, response = self._clients[sock] request.body.write(data) contentLength = int(request.headers.get("Content-Length", "0")) if not request.body.tell() == contentLength: return else: requestline, data = data.split("\n", 1) requestline = requestline.strip() method, path, protocol = requestline.split(" ", 2) scheme, location, path, params, qs, frag = urlparse(path) protocol = tuple(map(int, protocol[5:].split("."))) request = wrappers.Request(sock, method, scheme, path, protocol, qs) response = wrappers.Response(sock, request) self._clients[sock] = request, response if frag: error = HTTPError(request, response, 400) return self.push(error, "httperror", self.channel) if params: path = "%s;%s" % (path, params) # Unquote the path+params (e.g. "/this%20path" -> "this path"). # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 # # But note that "...a URI must be separated into its components # before the escaped characters within those components can be # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 path = "%2F".join(map(unquote, quoted_slash.split(path))) # Compare request and server HTTP protocol versions, in case our # server does not support the requested protocol. Limit our output # to min(req, server). We want the following output: # request server actual written supported response # protocol protocol response protocol feature set # a 1.0 1.0 1.0 1.0 # b 1.0 1.1 1.1 1.0 # c 1.1 1.0 1.0 1.0 # d 1.1 1.1 1.1 1.1 # Notice that, in (b), the response will be "HTTP/1.1" even though # the client only understands 1.0. RFC 2616 10.5.6 says we should # only return 505 if the _major_ version is different. if not request.protocol[0] == request.server_protocol[0]: error = HTTPError(request, response, 505) return self.push(error, "httperror", self.channel) rp = request.protocol sp = request.server_protocol response.protocol = "HTTP/%s.%s" % min(rp, sp) headers, body = parseHeaders(StringIO(data)) request.headers = headers request.body.write(body) if headers.get("Expect", "") == "100-continue": return self.simple(sock, 100) contentLength = int(headers.get("Content-Length", "0")) if not request.body.tell() == contentLength: return # Persistent connection support if request.protocol == (1, 1): # Both server and client are HTTP/1.1 if request.headers.get("Connection", "").lower() == "close": response.close = True else: # Either the server or client (or both) are HTTP/1.0 if request.headers.get("Connection", "").lower() != "keep-alive": response.close = True request.body.seek(0) self.push(Request(request, response), "request", "web")
def _fetch(self, url): """Fetch results from JSON endpoint and return as dict.""" f = urlfetch.fetch(url) if f.status_code != 200: raise HTTPError(f.status_code, f.headers) return json.loads(f.content)
def serve_file(request, response, path, type=None, disposition=None, name=None): """Set status, headers, and body in order to serve the given file. The Content-Type header will be set to the type arg, if provided. If not provided, the Content-Type will be guessed by the file extension of the 'path' argument. If disposition is not None, the Content-Disposition header will be set to "<disposition>; filename=<name>". If name is None, it will be set to the basename of path. If disposition is None, no Content-Disposition header will be written. """ if not os.path.isabs(path): raise ValueError("'%s' is not an absolute path." % path) try: st = os.stat(path) except OSError: return NotFound(request, response) # Check if path is a directory. if stat.S_ISDIR(st.st_mode): # Let the caller deal with it as they like. return NotFound(request, response) # Set the Last-Modified response header, so that # modified-since validation code can work. response.headers['Last-Modified'] = formatdate(st.st_mtime) validate_since(request, response) if type is None: # Set content-type based on filename extension ext = "" i = path.rfind('.') if i != -1: ext = path[i:].lower() type = mimetypes.types_map.get(ext, "text/plain") response.headers['Content-Type'] = type if disposition is not None: if name is None: name = os.path.basename(path) cd = '%s; filename="%s"' % (disposition, name) response.headers["Content-Disposition"] = cd # Set Content-Length and use an iterable (file object) # this way CP won't load the whole file in memory c_len = st.st_size bodyfile = open(path, 'rb') # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code if request.protocol >= (1, 1): response.headers["Accept-Ranges"] = "bytes" r = get_ranges(request.headers.get('Range'), c_len) if r == []: response.headers['Content-Range'] = "bytes */%s" % c_len message = "Invalid Range (first-byte-pos greater than Content-Length)" return HTTPError(request, response, 416, message) if r: if len(r) == 1: # Return a single-part response. start, stop = r[0] r_len = stop - start response.status = "206 Partial Content" response.headers['Content-Range'] = ("bytes %s-%s/%s" % (start, stop - 1, c_len)) response.headers['Content-Length'] = r_len bodyfile.seek(start) response.body = bodyfile.read(r_len) else: # Return a multipart/byteranges response. response.status = "206 Partial Content" boundary = mimetools.choose_boundary() ct = "multipart/byteranges; boundary=%s" % boundary response.headers['Content-Type'] = ct if response.headers.has_key("Content-Length"): # Delete Content-Length header so finalize() recalcs it. del response.headers["Content-Length"] def file_ranges(): # Apache compatibility: yield "\r\n" for start, stop in r: yield "--" + boundary yield "\r\nContent-type: %s" % type yield ("\r\nContent-range: bytes %s-%s/%s\r\n\r\n" % (start, stop - 1, c_len)) bodyfile.seek(start) yield bodyfile.read(stop - start) yield "\r\n" # Final boundary yield "--" + boundary + "--" # Apache compatibility: yield "\r\n" response.body = file_ranges() else: response.headers['Content-Length'] = c_len response.body = bodyfile else: response.headers['Content-Length'] = c_len response.body = bodyfile return response
class Browser: """A class for making calls to lazr.restful web services.""" NOT_MODIFIED = object() MAX_RETRIES = 6 def __init__(self, service_root, credentials, cache=None, timeout=None, proxy_info=None, user_agent=None, max_retries=MAX_RETRIES): """Initialize, possibly creating a cache. If no cache is provided, a temporary directory will be used as a cache. The temporary directory will be automatically removed when the Python process exits. """ if cache is None: cache = tempfile.mkdtemp() atexit.register(shutil.rmtree, cache) if isinstance(cache, basestring): cache = MultipleRepresentationCache(cache) self._connection = service_root.httpFactory( credentials, cache, timeout, proxy_info) self.user_agent = user_agent self.max_retries = max_retries def _request_and_retry(self, url, method, body, headers): for retry_count in range(0, self.max_retries+1): response, content = self._connection.request( url, method=method, body=body, headers=headers) if (response.status in [502, 503] and retry_count < self.max_retries): # The server returned a 502 or 503. Sleep for 0, 1, 2, # 4, 8, 16, ... seconds and try again. sleep_for = int(2**(retry_count-1)) sleep(sleep_for) else: break # Either the request succeeded or we gave up. return response, content def _request(self, url, data=None, method='GET', media_type='application/json', extra_headers=None): """Create an authenticated request object.""" # If the user is trying to get data that has been redacted, # give a helpful message. if url == "tag:launchpad.net:2008:redacted": raise ValueError("You tried to access a resource that you " "don't have the server-side permission to see.") # Add extra headers for the request. headers = {'Accept' : media_type} if self.user_agent is not None: headers['User-Agent'] = self.user_agent if isinstance(self._connection.cache, MultipleRepresentationCache): self._connection.cache.request_media_type = media_type if extra_headers is not None: headers.update(extra_headers) # Make the request. try: response, content = self._request_and_retry( str(url), method=method, body=data, headers=headers) except SSLHandshakeError, e: msg = str(e) if "SSL3_GET_SERVER_CERTIFICATE:certificate verify failed" in msg: raise SSLHandshakeError( "%s: perhaps set LP_DISABLE_SSL_CERTIFICATE_VALIDATION if " "certificate validation failed and you genuinely trust the " "remote server." % (e,)) raise if response.status == 304: # The resource didn't change. if content == '': if ('If-None-Match' in headers or 'If-Modified-Since' in headers): # The caller made a conditional request, and the # condition failed. Rather than send an empty # representation, which might be misinterpreted, # send a special object that will let the calling code know # that the resource was not modified. return response, self.NOT_MODIFIED else: # The caller didn't make a conditional request, # but the response code is 304 and there's no # content. The only way to handle this is to raise # an error. # # We don't use error_for() here because 304 is not # normally considered an error condition. raise HTTPError(response, content) else: # XXX leonardr 2010/04/12 bug=httplib2#97 # # Why is this check here? Why would there ever be any # content when the response code is 304? It's because of # an httplib2 bug that sometimes sets a 304 response # code when caching retrieved documents. When the # cached document is retrieved, we get a 304 response # code and a full representation. # # Since the cache lookup succeeded, the 'real' # response code is 200. This code undoes the bad # behavior in httplib2. response.status = 200 return response, content # Turn non-2xx responses into appropriate HTTPError subclasses. error = error_for(response, content) if error is not None: raise error return response, content