def get_authenticated_user(self, callback, http_client=None): """Gets the OAuth authorized user and access token on callback. This method should be called from the handler for your registered OAuth Callback URL to complete the registration process. We call callback with the authenticated user, which in addition to standard attributes like 'name' includes the 'access_key' attribute, which contains the OAuth access you can use to make authorized requests to this service on behalf of the user. """ request_key = escape.utf8(self.get_argument("oauth_token")) oauth_verifier = self.get_argument("oauth_verifier", None) request_cookie = self.get_cookie("_oauth_request_token") if not request_cookie: logging.warning("Missing OAuth request token cookie") callback(None) return self.clear_cookie("_oauth_request_token") cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")] if cookie_key != request_key: logging.info((cookie_key, request_key, request_cookie)) logging.warning("Request token does not match cookie") callback(None) return token = dict(key=cookie_key, secret=cookie_secret) if oauth_verifier: token["verifier"] = oauth_verifier if http_client is None: http_client = httpclient.AsyncHTTPClient() http_client.fetch(self._oauth_access_token_url(token), self.async_callback(self._on_access_token, callback))
def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None): """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request. See http://oauth.net/core/1.0a/#signing_process """ parts = urlparse.urlparse(url) scheme, netloc, path = parts[:3] normalized_url = scheme.lower() + "://" + netloc.lower() + path base_elems = [] base_elems.append(method.upper()) base_elems.append(normalized_url) base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items()))) base_string = "&".join(_oauth_escape(e) for e in base_elems) key_elems = [escape.utf8(urllib.quote(consumer_token["secret"], safe='~'))] key_elems.append( escape.utf8(urllib.quote(token["secret"], safe='~') if token else "")) key = b("&").join(key_elems) hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) return binascii.b2a_base64(hash.digest())[:-1]
def get_authenticated_user(self, callback, http_client=None): """Gets the OAuth authorized user and access token on callback. This method should be called from the handler for your registered OAuth Callback URL to complete the registration process. We call callback with the authenticated user, which in addition to standard attributes like 'name' includes the 'access_key' attribute, which contains the OAuth access you can use to make authorized requests to this service on behalf of the user. """ request_key = escape.utf8(self.get_argument("oauth_token")) oauth_verifier = self.get_argument("oauth_verifier", None) request_cookie = self.get_cookie("_oauth_request_token") if not request_cookie: logging.warning("Missing OAuth request token cookie") callback(None) return self.clear_cookie("_oauth_request_token") cookie_key, cookie_secret = [ base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|") ] if cookie_key != request_key: logging.info((cookie_key, request_key, request_cookie)) logging.warning("Request token does not match cookie") callback(None) return token = dict(key=cookie_key, secret=cookie_secret) if oauth_verifier: token["verifier"] = oauth_verifier if http_client is None: http_client = httpclient.AsyncHTTPClient() http_client.fetch(self._oauth_access_token_url(token), self.async_callback(self._on_access_token, callback))
def _on_connect(self, parsed): if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, self._on_timeout) if (self.request.validate_cert and isinstance(self.stream, SSLIOStream)): match_hostname(self.stream.socket.getpeercert(), parsed.hostname) if (self.request.method not in self._SUPPORTED_METHODS and not self.request.allow_nonstandard_methods): raise KeyError("unknown method %s" % self.request.method) for key in ('network_interface', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'): if getattr(self.request, key, None): raise NotImplementedError('%s not supported' % key) if "Host" not in self.request.headers: self.request.headers["Host"] = parsed.netloc username, password = None, None if parsed.username is not None: username, password = parsed.username, parsed.password elif self.request.auth_username is not None: username = self.request.auth_username password = self.request.auth_password if username is not None: auth = utf8(username) + b(":") + utf8(password) self.request.headers["Authorization"] = (b("Basic ") + base64.b64encode(auth)) if self.request.user_agent: self.request.headers["User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: if self.request.method in ("POST", "PUT"): assert self.request.body is not None else: assert self.request.body is None if self.request.body is not None: self.request.headers["Content-Length"] = str(len( self.request.body)) if (self.request.method == "POST" and "Content-Type" not in self.request.headers): self.request.headers[ "Content-Type"] = "application/x-www-form-urlencoded" if self.request.use_gzip: self.request.headers["Accept-Encoding"] = "gzip" req_path = ((parsed.path or '/') + (('?' + parsed.query) if parsed.query else '')) request_lines = [ utf8("%s %s HTTP/1.0" % (self.request.method, req_path)) ] for k, v in self.request.headers.get_all(): line = utf8(k) + b(": ") + utf8(v) if b('\n') in line: raise ValueError('Newline in header: ' + repr(line)) request_lines.append(line) self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n")) if self.request.body is not None: self.stream.write(self.request.body) self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)
def __call__(self, request): data = {} response = [] def start_response(status, response_headers, exc_info=None): data["status"] = status data["headers"] = response_headers return response.append app_response = self.wsgi_application( WSGIContainer.environ(request), start_response) response.extend(app_response) body = b("").join(response) if hasattr(app_response, "close"): app_response.close() if not data: raise Exception("WSGI app did not call start_response") status_code = int(data["status"].split()[0]) headers = data["headers"] header_set = set(k.lower() for (k,v) in headers) body = escape.utf8(body) if "content-length" not in header_set: headers.append(("Content-Length", str(len(body)))) if "content-type" not in header_set: headers.append(("Content-Type", "text/html; charset=UTF-8")) if "server" not in header_set: headers.append(("Server", "Anzu/%s" % anzu.version)) parts = [escape.utf8("HTTP/1.1 " + data["status"] + "\r\n")] for key, value in headers: parts.append(escape.utf8(key) + b(": ") + escape.utf8(value) + b("\r\n")) parts.append(b("\r\n")) parts.append(body) request.write(b("").join(parts)) request.finish() self._log(status_code, request)
def _on_connect(self, parsed): if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, self._on_timeout) if (self.request.validate_cert and isinstance(self.stream, SSLIOStream)): match_hostname(self.stream.socket.getpeercert(), parsed.hostname) if (self.request.method not in self._SUPPORTED_METHODS and not self.request.allow_nonstandard_methods): raise KeyError("unknown method %s" % self.request.method) for key in ('network_interface', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'): if getattr(self.request, key, None): raise NotImplementedError('%s not supported' % key) if "Host" not in self.request.headers: self.request.headers["Host"] = parsed.netloc username, password = None, None if parsed.username is not None: username, password = parsed.username, parsed.password elif self.request.auth_username is not None: username = self.request.auth_username password = self.request.auth_password if username is not None: auth = utf8(username) + b(":") + utf8(password) self.request.headers["Authorization"] = (b("Basic ") + base64.b64encode(auth)) if self.request.user_agent: self.request.headers["User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: if self.request.method in ("POST", "PUT"): assert self.request.body is not None else: assert self.request.body is None if self.request.body is not None: self.request.headers["Content-Length"] = str(len( self.request.body)) if (self.request.method == "POST" and "Content-Type" not in self.request.headers): self.request.headers["Content-Type"] = "application/x-www-form-urlencoded" if self.request.use_gzip: self.request.headers["Accept-Encoding"] = "gzip" req_path = ((parsed.path or '/') + (('?' + parsed.query) if parsed.query else '')) request_lines = [utf8("%s %s HTTP/1.0" % (self.request.method, req_path))] for k, v in self.request.headers.get_all(): line = utf8(k) + b(": ") + utf8(v) if b('\n') in line: raise ValueError('Newline in header: ' + repr(line)) request_lines.append(line) self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n")) if self.request.body is not None: self.stream.write(self.request.body) self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)
def render_xml(self, value): assert isinstance(value, dict) and len(value) == 1 self.set_header("Content-Type", "application/xml; charset=UTF-8") name = value.keys()[0] parts = [] parts.append('<' + escape.utf8(name) + ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') self._render_parts(value.values()[0], parts) parts.append('</' + escape.utf8(name) + '>') self.finish('<?xml version="1.0" encoding="UTF-8"?>\n' + ''.join(parts))
def test_xhtml_escape(self): tests = [ ("<foo>", "<foo>"), (u"<foo>", u"<foo>"), (b("<foo>"), b("<foo>")), ("<>&\"", "<>&""), ("&", "&amp;"), ] for unescaped, escaped in tests: self.assertEqual(utf8(xhtml_escape(unescaped)), utf8(escaped)) self.assertEqual(utf8(unescaped), utf8(xhtml_unescape(escaped)))
def test_unicode_literal_expression(self): # Unicode literals should be usable in templates. Note that this # test simulates unicode characters appearing directly in the # template file (with utf8 encoding), i.e. \u escapes would not # be used in the template file itself. if str is unicode: # python 3 needs a different version of this test since # 2to3 doesn't run on template internals template = Template(utf8(u'{{ "\u00e9" }}')) else: template = Template(utf8(u'{{ u"\u00e9" }}')) self.assertEqual(template.generate(), utf8(u"\u00e9"))
def test_url_unescape(self): tests = [ ('%C3%A9', u'\u00e9', 'utf8'), ('%C3%A9', u'\u00c3\u00a9', 'latin1'), ('%C3%A9', utf8(u'\u00e9'), None), ] for escaped, unescaped, encoding in tests: # input strings to url_unescape should only contain ascii # characters, but make sure the function accepts both byte # and unicode strings. self.assertEqual(url_unescape(to_unicode(escaped), encoding), unescaped) self.assertEqual(url_unescape(utf8(escaped), encoding), unescaped)
def test_special_filenames(self): filenames = [ 'a;b.txt', 'a"b.txt', 'a";b.txt', 'a;"b.txt', 'a";";.txt', 'a\\"b.txt', 'a\\b.txt', ] for filename in filenames: logging.info("trying filename %r", filename) data = """\ --1234 Content-Disposition: form-data; name="files"; filename="%s" Foo --1234--""" % filename.replace('\\', '\\\\').replace('"', '\\"') data = utf8(data.replace("\n", "\r\n")) args = {} files = {} parse_multipart_form_data(b("1234"), data, args, files) file = files["files"][0] self.assertEqual(file["filename"], filename) self.assertEqual(file["body"], b("Foo"))
def environ(request): """Converts a `anzu.httpserver.HTTPRequest` to a WSGI environment. """ hostport = request.host.split(":") if len(hostport) == 2: host = hostport[0] port = int(hostport[1]) else: host = request.host port = 443 if request.protocol == "https" else 80 environ = { "REQUEST_METHOD": request.method, "SCRIPT_NAME": "", "PATH_INFO": urllib.unquote(request.path), "QUERY_STRING": request.query, "REMOTE_ADDR": request.remote_ip, "SERVER_NAME": host, "SERVER_PORT": str(port), "SERVER_PROTOCOL": request.version, "wsgi.version": (1, 0), "wsgi.url_scheme": request.protocol, "wsgi.input": BytesIO(escape.utf8(request.body)), "wsgi.errors": sys.stderr, "wsgi.multithread": False, "wsgi.multiprocess": True, "wsgi.run_once": False, } if "Content-Type" in request.headers: environ["CONTENT_TYPE"] = request.headers.pop("Content-Type") if "Content-Length" in request.headers: environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length") for key, value in request.headers.iteritems(): environ["HTTP_" + key.replace("-", "_").upper()] = value return environ
def _render_parts(self, value, parts=[]): if isinstance(value, (unicode, bytes_type)): parts.append(escape.xhtml_escape(value)) elif isinstance(value, int) or isinstance(value, long): parts.append(str(value)) elif isinstance(value, datetime.datetime): parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) elif isinstance(value, dict): for name, subvalue in value.iteritems(): if not isinstance(subvalue, list): subvalue = [subvalue] for subsubvalue in subvalue: parts.append('<' + escape.utf8(name) + '>') self._render_parts(subsubvalue, parts) parts.append('</' + escape.utf8(name) + '>') else: raise Exception("Unknown S3 value type %r", value)
def test_json_decode(self): # json_decode accepts both bytes and unicode, but strings it returns # are always unicode. self.assertEqual(json_decode(b('"foo"')), u"foo") self.assertEqual(json_decode(u'"foo"'), u"foo") # Non-ascii bytes are interpreted as utf8 self.assertEqual(json_decode(utf8(u'"\u00e9"')), u"\u00e9")
def __init__(self, environ): """Parses the given WSGI environ to construct the request.""" self.method = environ["REQUEST_METHOD"] self.path = urllib.quote(environ.get("SCRIPT_NAME", "")) self.path += urllib.quote(environ.get("PATH_INFO", "")) self.uri = self.path self.arguments = {} self.query = environ.get("QUERY_STRING", "") if self.query: self.uri += "?" + self.query arguments = cgi.parse_qs(self.query) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self.arguments[name] = values self.version = "HTTP/1.1" self.headers = httputil.HTTPHeaders() if environ.get("CONTENT_TYPE"): self.headers["Content-Type"] = environ["CONTENT_TYPE"] if environ.get("CONTENT_LENGTH"): self.headers["Content-Length"] = environ["CONTENT_LENGTH"] for key in environ: if key.startswith("HTTP_"): self.headers[key[5:].replace("_", "-")] = environ[key] if self.headers.get("Content-Length"): self.body = environ["wsgi.input"].read( int(self.headers["Content-Length"])) else: self.body = "" self.protocol = environ["wsgi.url_scheme"] self.remote_ip = environ.get("REMOTE_ADDR", "") if environ.get("HTTP_HOST"): self.host = environ["HTTP_HOST"] else: self.host = environ["SERVER_NAME"] # Parse request body self.files = {} content_type = self.headers.get("Content-Type", "") if content_type.startswith("application/x-www-form-urlencoded"): for name, values in cgi.parse_qs(self.body).iteritems(): self.arguments.setdefault(name, []).extend(values) elif content_type.startswith("multipart/form-data"): if 'boundary=' in content_type: boundary = content_type.split('boundary=', 1)[1] if boundary: httputil.parse_multipart_form_data(utf8(boundary), self.body, self.arguments, self.files) else: logging.warning("Invalid multipart/form-data") self._start_time = time.time() self._finish_time = None
def raw_fetch(self, headers, body): conn = RawRequestHTTPConnection( self.io_loop, self.http_client, httpclient.HTTPRequest(self.get_url("/")), None, self.stop, 1024 * 1024) conn.set_request( b("\r\n").join(headers + [utf8("Content-Length: %d\r\n" % len(body))]) + b("\r\n") + body) response = self.wait() response.rethrow() return response
def raw_fetch(self, headers, body): conn = RawRequestHTTPConnection(self.io_loop, self.http_client, httpclient.HTTPRequest(self.get_url("/")), None, self.stop, 1024*1024) conn.set_request( b("\r\n").join(headers + [utf8("Content-Length: %d\r\n" % len(body))]) + b("\r\n") + body) response = self.wait() response.rethrow() return response
def __init__(self, environ): """Parses the given WSGI environ to construct the request.""" self.method = environ["REQUEST_METHOD"] self.path = urllib.quote(environ.get("SCRIPT_NAME", "")) self.path += urllib.quote(environ.get("PATH_INFO", "")) self.uri = self.path self.arguments = {} self.query = environ.get("QUERY_STRING", "") if self.query: self.uri += "?" + self.query arguments = cgi.parse_qs(self.query) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self.arguments[name] = values self.version = "HTTP/1.1" self.headers = httputil.HTTPHeaders() if environ.get("CONTENT_TYPE"): self.headers["Content-Type"] = environ["CONTENT_TYPE"] if environ.get("CONTENT_LENGTH"): self.headers["Content-Length"] = environ["CONTENT_LENGTH"] for key in environ: if key.startswith("HTTP_"): self.headers[key[5:].replace("_", "-")] = environ[key] if self.headers.get("Content-Length"): self.body = environ["wsgi.input"].read( int(self.headers["Content-Length"])) else: self.body = "" self.protocol = environ["wsgi.url_scheme"] self.remote_ip = environ.get("REMOTE_ADDR", "") if environ.get("HTTP_HOST"): self.host = environ["HTTP_HOST"] else: self.host = environ["SERVER_NAME"] # Parse request body self.files = {} content_type = self.headers.get("Content-Type", "") if content_type.startswith("application/x-www-form-urlencoded"): for name, values in cgi.parse_qs(self.body).iteritems(): self.arguments.setdefault(name, []).extend(values) elif content_type.startswith("multipart/form-data"): if 'boundary=' in content_type: boundary = content_type.split('boundary=',1)[1] if boundary: httputil.parse_multipart_form_data( utf8(boundary), self.body, self.arguments, self.files) else: logging.warning("Invalid multipart/form-data") self._start_time = time.time() self._finish_time = None
def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None): """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request. See http://oauth.net/core/1.0a/#signing_process """ parts = urlparse.urlparse(url) scheme, netloc, path = parts[:3] normalized_url = scheme.lower() + "://" + netloc.lower() + path base_elems = [] base_elems.append(method.upper()) base_elems.append(normalized_url) base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items()))) base_string = "&".join(_oauth_escape(e) for e in base_elems) key_elems = [escape.utf8(urllib.quote(consumer_token["secret"], safe='~'))] key_elems.append(escape.utf8(urllib.quote(token["secret"], safe='~') if token else "")) key = b("&").join(key_elems) hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) return binascii.b2a_base64(hash.digest())[:-1]
def __call__(self, request): data = {} response = [] def start_response(status, response_headers, exc_info=None): data["status"] = status data["headers"] = response_headers return response.append app_response = self.wsgi_application(WSGIContainer.environ(request), start_response) response.extend(app_response) body = b("").join(response) if hasattr(app_response, "close"): app_response.close() if not data: raise Exception("WSGI app did not call start_response") status_code = int(data["status"].split()[0]) headers = data["headers"] header_set = set(k.lower() for (k, v) in headers) body = escape.utf8(body) if "content-length" not in header_set: headers.append(("Content-Length", str(len(body)))) if "content-type" not in header_set: headers.append(("Content-Type", "text/html; charset=UTF-8")) if "server" not in header_set: headers.append(("Server", "Anzu/%s" % anzu.version)) parts = [escape.utf8("HTTP/1.1 " + data["status"] + "\r\n")] for key, value in headers: parts.append( escape.utf8(key) + b(": ") + escape.utf8(value) + b("\r\n")) parts.append(b("\r\n")) parts.append(body) request.write(b("").join(parts)) request.finish() self._log(status_code, request)
def test_cookie_special_char(self): response = self.fetch("/special_char") headers = response.headers.get_list("Set-Cookie") self.assertEqual(len(headers), 3) self.assertEqual(headers[0], 'equals="a=b"; Path=/') # python 2.7 octal-escapes the semicolon; older versions leave it alone self.assertTrue(headers[1] in ('semicolon="a;b"; Path=/', 'semicolon="a\\073b"; Path=/'), headers[1]) self.assertEqual(headers[2], 'quote="a\\"b"; Path=/') data = [('foo=a=b', 'a=b'), ('foo="a=b"', 'a=b'), ('foo="a;b"', 'a;b'), #('foo=a\\073b', 'a;b'), # even encoded, ";" is a delimiter ('foo="a\\073b"', 'a;b'), ('foo="a\\"b"', 'a"b'), ] for header, expected in data: logging.info("trying %r", header) response = self.fetch("/get", headers={"Cookie": header}) self.assertEqual(response.body, utf8(expected))
def _on_request_body(self, data): self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PUT"): if content_type.startswith("application/x-www-form-urlencoded"): arguments = parse_qs_bytes(native_str(self._request.body)) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self._request.arguments.setdefault(name, []).extend(values) elif content_type.startswith("multipart/form-data"): fields = content_type.split(";") for field in fields: k, sep, v = field.strip().partition("=") if k == "boundary" and v: httputil.parse_multipart_form_data( utf8(v), data, self._request.arguments, self._request.files) break else: logging.warning("Invalid multipart/form-data") self.request_callback(self._request)
def test_cookie_tampering_future_timestamp(self): handler = CookieTestRequestHandler() # this string base64-encodes to '12345678' handler.set_secure_cookie('foo', binascii.a2b_hex(b('d76df8e7aefc'))) cookie = handler._cookies['foo'] match = re.match(b(r'12345678\|([0-9]+)\|([0-9a-f]+)'), cookie) assert match timestamp = match.group(1) sig = match.group(2) self.assertEqual( _create_signature(handler.application.settings["cookie_secret"], 'foo', '12345678', timestamp), sig) # shifting digits from payload to timestamp doesn't alter signature # (this is not desirable behavior, just confirming that that's how it # works) self.assertEqual( _create_signature(handler.application.settings["cookie_secret"], 'foo', '1234', b('5678') + timestamp), sig) # tamper with the cookie handler._cookies['foo'] = utf8('1234|5678%s|%s' % (timestamp, sig)) # it gets rejected assert handler.get_secure_cookie('foo') is None
def test_cookie_special_char(self): response = self.fetch("/special_char") headers = response.headers.get_list("Set-Cookie") self.assertEqual(len(headers), 3) self.assertEqual(headers[0], 'equals="a=b"; Path=/') # python 2.7 octal-escapes the semicolon; older versions leave it alone self.assertTrue( headers[1] in ('semicolon="a;b"; Path=/', 'semicolon="a\\073b"; Path=/'), headers[1]) self.assertEqual(headers[2], 'quote="a\\"b"; Path=/') data = [ ('foo=a=b', 'a=b'), ('foo="a=b"', 'a=b'), ('foo="a;b"', 'a;b'), #('foo=a\\073b', 'a;b'), # even encoded, ";" is a delimiter ('foo="a\\073b"', 'a;b'), ('foo="a\\"b"', 'a"b'), ] for header, expected in data: logging.info("trying %r", header) response = self.fetch("/get", headers={"Cookie": header}) self.assertEqual(response.body, utf8(expected))
def test_special_filenames(self): filenames = ['a;b.txt', 'a"b.txt', 'a";b.txt', 'a;"b.txt', 'a";";.txt', 'a\\"b.txt', 'a\\b.txt', ] for filename in filenames: logging.info("trying filename %r", filename) data = """\ --1234 Content-Disposition: form-data; name="files"; filename="%s" Foo --1234--""" % filename.replace('\\', '\\\\').replace('"', '\\"') data = utf8(data.replace("\n", "\r\n")) args = {} files = {} parse_multipart_form_data(b("1234"), data, args, files) file = files["files"][0] self.assertEqual(file["filename"], filename) self.assertEqual(file["body"], b("Foo"))
def test_body_encoding(self): unicode_body = u"\xe9" byte_body = binascii.a2b_hex(b("e9")) # unicode string in body gets converted to utf8 response = self.fetch("/echopost", method="POST", body=unicode_body, headers={"Content-Type": "application/blah"}) self.assertEqual(response.headers["Content-Length"], "2") self.assertEqual(response.body, utf8(unicode_body)) # byte strings pass through directly response = self.fetch("/echopost", method="POST", body=byte_body, headers={"Content-Type": "application/blah"}) self.assertEqual(response.headers["Content-Length"], "1") self.assertEqual(response.body, byte_body) # Mixing unicode in headers and byte string bodies shouldn't # break anything response = self.fetch("/echopost", method="POST", body=byte_body, headers={"Content-Type": "application/blah"}, user_agent=u"foo") self.assertEqual(response.headers["Content-Length"], "1") self.assertEqual(response.body, byte_body)
def test_bytes(self): template = Template("Hello {{ name }}!") self.assertEqual(template.generate(name=utf8("Ben")), b("Hello Ben!"))
def test_absolute_static_url(self): response = self.fetch("/abs_static_url/robots.txt") self.assertEqual(response.body, utf8(self.get_url("/") + "static/robots.txt?v=f71d2"))
def test_if(self): template = Template(utf8("{% if x > 4 %}yes{% else %}no{% end %}")) self.assertEqual(template.generate(x=5), b("yes")) self.assertEqual(template.generate(x=3), b("no"))
def test_json_encode(self): # json deals with strings, not bytes, but our encoding function should # accept bytes as well as long as they are utf8. self.assertEqual(json_decode(json_encode(u"\u00e9")), u"\u00e9") self.assertEqual(json_decode(json_encode(utf8(u"\u00e9"))), u"\u00e9") self.assertRaises(UnicodeDecodeError, json_encode, b("\xe9"))
def test_comment(self): template = Template(utf8("{% comment blah blah %}foo")) self.assertEqual(template.generate(), b("foo"))
def test_apply(self): def upper(s): return s.upper() template = Template(utf8("{% apply upper %}foo{% end %}")) self.assertEqual(template.generate(upper=upper), b("FOO"))
def test_unicode_template(self): template = Template(utf8(u"\u00e9")) self.assertEqual(template.generate(), utf8(u"\u00e9"))
def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.URL, request.url) # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, # Google's OpenID endpoint). Additionally, this behavior has # a bug in conjunction with the curl_multi_socket_action API # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), # which increases the delays. It's more trouble than it's worth, # so just turn off the feature (yes, setting Expect: to an empty # value is the official way to disable this) if "Expect" not in request.headers: request.headers["Expect"] = "" # libcurl adds Pragma: no-cache by default; disable that too if "Pragma" not in request.headers: request.headers["Pragma"] = "" # Request headers may be either a regular dict or HTTPHeaders object if isinstance(request.headers, httputil.HTTPHeaders): curl.setopt(pycurl.HTTPHEADER, [utf8("%s: %s" % i) for i in request.headers.get_all()]) else: curl.setopt(pycurl.HTTPHEADER, [utf8("%s: %s" % i) for i in request.headers.iteritems()]) if request.header_callback: curl.setopt(pycurl.HEADERFUNCTION, request.header_callback) else: curl.setopt(pycurl.HEADERFUNCTION, lambda line: _curl_header_callback(headers, line)) if request.streaming_callback: curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback) else: curl.setopt(pycurl.WRITEFUNCTION, buffer.write) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout)) curl.setopt(pycurl.TIMEOUT, int(request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, utf8(request.user_agent)) else: curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: curl.setopt(pycurl.INTERFACE, request.network_interface) if request.use_gzip: curl.setopt(pycurl.ENCODING, "gzip,deflate") else: curl.setopt(pycurl.ENCODING, "none") if request.proxy_host and request.proxy_port: curl.setopt(pycurl.PROXY, request.proxy_host) curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: credentials = '%s:%s' % (request.proxy_username, request.proxy_password) curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) else: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) if request.ca_certs is not None: curl.setopt(pycurl.CAINFO, request.ca_certs) else: # There is no way to restore pycurl.CAINFO to its default value # (Using unsetopt makes it reject all certificates). # I don't see any way to read the default value from python so it # can be restored later. We'll have to just leave CAINFO untouched # if no ca_certs file was specified, and require that if any # request uses a custom ca_certs file, they all must. pass if request.allow_ipv6 is False: # Curl behaves reasonably when DNS resolution gives an ipv6 address # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) # Set the request method through curl's retarded interface which makes # up names for almost every single method curl_options = { "GET": pycurl.HTTPGET, "POST": pycurl.POST, "PUT": pycurl.UPLOAD, "HEAD": pycurl.NOBODY, } custom_methods = set(["DELETE"]) for o in curl_options.values(): curl.setopt(o, False) if request.method in curl_options: curl.unsetopt(pycurl.CUSTOMREQUEST) curl.setopt(curl_options[request.method], True) elif request.allow_nonstandard_methods or request.method in custom_methods: curl.setopt(pycurl.CUSTOMREQUEST, request.method) else: raise KeyError('unknown method ' + request.method) # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): request_buffer = cStringIO.StringIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": def ioctl(cmd): if cmd == curl.IOCMD_RESTARTREAD: request_buffer.seek(0) curl.setopt(pycurl.IOCTLFUNCTION, ioctl) curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) if request.auth_username and request.auth_password: userpwd = "%s:%s" % (request.auth_username, request.auth_password) curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) curl.setopt(pycurl.USERPWD, userpwd) logging.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) else: curl.unsetopt(pycurl.USERPWD) logging.debug("%s %s", request.method, request.url) if request.client_key is not None or request.client_cert is not None: raise ValueError("Client certificate not supported with curl_httpclient") if threading.activeCount() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads # are used, signals should be disabled. This has the side effect # of disabling DNS timeouts in some environments (when libcurl is # not linked against ares), so we don't do it when there is only one # thread. Applications that use many short-lived threads may need # to set NOSIGNAL manually in a prepare_curl_callback since # there may not be any other threads running at the time we call # threading.activeCount. curl.setopt(pycurl.NOSIGNAL, 1) if request.prepare_curl_callback is not None: request.prepare_curl_callback(curl)
def __init__(self, url, method="GET", headers=None, body=None, auth_username=None, auth_password=None, connect_timeout=20.0, request_timeout=20.0, if_modified_since=None, follow_redirects=True, max_redirects=5, user_agent=None, use_gzip=True, network_interface=None, streaming_callback=None, header_callback=None, prepare_curl_callback=None, proxy_host=None, proxy_port=None, proxy_username=None, proxy_password='', allow_nonstandard_methods=False, validate_cert=True, ca_certs=None, allow_ipv6=None, client_key=None, client_cert=None): """Creates an `HTTPRequest`. All parameters except `url` are optional. :arg string url: URL to fetch :arg string method: HTTP method, e.g. "GET" or "POST" :arg headers: Additional HTTP headers to pass on the request :type headers: `~anzu.httputil.HTTPHeaders` or `dict` :arg string auth_username: Username for HTTP "Basic" authentication :arg string auth_password: Password for HTTP "Basic" authentication :arg float connect_timeout: Timeout for initial connection in seconds :arg float request_timeout: Timeout for entire request in seconds :arg datetime if_modified_since: Timestamp for ``If-Modified-Since`` header :arg bool follow_redirects: Should redirects be followed automatically or return the 3xx response? :arg int max_redirects: Limit for `follow_redirects` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server :arg string network_interface: Network interface to use for request :arg callable streaming_callback: If set, `streaming_callback` will be run with each chunk of data as it is received, and `~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in the final response. :arg callable header_callback: If set, `header_callback` will be run with each header line as it is received, and `~HTTPResponse.headers` will be empty in the final response. :arg callable prepare_curl_callback: If set, will be called with a `pycurl.Curl` object to allow the application to make additional `setopt` calls. :arg string proxy_host: HTTP proxy hostname. To use proxies, `proxy_host` and `proxy_port` must be set; `proxy_username` and `proxy_pass` are optional. Proxies are currently only support with `curl_httpclient`. :arg int proxy_port: HTTP proxy port :arg string proxy_username: HTTP proxy username :arg string proxy_password: HTTP proxy password :arg bool allow_nonstandard_methods: Allow unknown values for `method` argument? :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, or None to use defaults. Note that in `curl_httpclient`, if any request uses a custom `ca_certs` file, they all must (they don't have to all use the same `ca_certs`, but it's not possible to mix requests with ca_certs and requests that use the defaults. :arg bool allow_ipv6: Use IPv6 when available? Default is false in `simple_httpclient` and true in `curl_httpclient` :arg string client_key: Filename for client SSL key, if any :arg string client_cert: Filename for client SSL certificate, if any """ if headers is None: headers = httputil.HTTPHeaders() if if_modified_since: timestamp = calendar.timegm(if_modified_since.utctimetuple()) headers["If-Modified-Since"] = email.utils.formatdate( timestamp, localtime=False, usegmt=True) self.proxy_host = proxy_host self.proxy_port = proxy_port self.proxy_username = proxy_username self.proxy_password = proxy_password self.url = url self.method = method self.headers = headers self.body = utf8(body) self.auth_username = auth_username self.auth_password = auth_password self.connect_timeout = connect_timeout self.request_timeout = request_timeout self.follow_redirects = follow_redirects self.max_redirects = max_redirects self.user_agent = user_agent self.use_gzip = use_gzip self.network_interface = network_interface self.streaming_callback = streaming_callback self.header_callback = header_callback self.prepare_curl_callback = prepare_curl_callback self.allow_nonstandard_methods = allow_nonstandard_methods self.validate_cert = validate_cert self.ca_certs = ca_certs self.allow_ipv6 = allow_ipv6 self.client_key = client_key self.client_cert = client_cert self.start_time = time.time()