def _load_static_table():
"""Parses the hpack static table, which was copied from
http://http2.github.io/http2-spec/compression.html#static.table
corresponding to
http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-12#appendix-A
"""
# start the table with a dummy entry 0
table = [None]
with open(os.path.join(os.path.dirname(__file__),
'hpack_static_table.txt')) as f:
for line in f:
if not line:
continue
fields = line.split('\t')
if int(fields[0]) != len(table):
raise ValueError("inconsistent numbering in static table")
name = utf8(fields[1].strip())
value = utf8(fields[2].strip()) if len(fields) > 2 else None
table.append((name, value))
static_keys = {}
static_pairs = {}
for i, pair in enumerate(table):
if pair is None:
continue
if pair[0] not in static_keys:
# For repeated keys, prefer the earlier one.
static_keys[pair[0]] = i
static_pairs[pair] = i
return table, static_keys, static_pairs
def get_authenticated_user(self, callback, http_client=None):
"""Gets the OAuth authorized user and access token on callback.
This method should be called from the handler for your registered
OAuth Callback URL to complete the registration process. We call
callback with the authenticated user, which in addition to standard
attributes like 'name' includes the 'access_key' attribute, which
contains the OAuth access you can use to make authorized requests
to this service on behalf of the user.
"""
request_key = escape.utf8(self.get_argument("oauth_token"))
oauth_verifier = self.get_argument("oauth_verifier", None)
request_cookie = self.get_cookie("_oauth_request_token")
if not request_cookie:
logging.warning("Missing OAuth request token cookie")
callback(None)
return
self.clear_cookie("_oauth_request_token")
cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")]
if cookie_key != request_key:
logging.info((cookie_key, request_key, request_cookie))
logging.warning("Request token does not match cookie")
callback(None)
return
token = dict(key=cookie_key, secret=cookie_secret)
if oauth_verifier:
token["verifier"] = oauth_verifier
if http_client is None:
http_client = httpclient.AsyncHTTPClient()
http_client.fetch(self._oauth_access_token_url(token),
self.async_callback(self._on_access_token, callback))
def fix_top_recommend_users():
thumb_up_recorder = UserAnswerThumbUpRecorder()
db = mongoclient.fbt
for exp in db.answers.find({}):
tags_with_class = [utf8(exp["class2"]) + ":" + utf8(tag) for tag in exp["tags"]]
if exp["thumb_up_num"] > 0:
thumb_up_recorder.thumb_up_for_user(exp["publisher"], tags_with_class, exp["thumb_up_num"])
def post(self):
data = self.post_schema()
name = data['name']
password = data['password']
try:
user = User.objects(name=name).get()
except DoesNotExist:
# 失败
self.write_not_found_entity_response()
# self.render("login.html", error="name not found")
return
hashed_password = bcrypt.hashpw(utf8(password),
utf8(user.hashed_password))
if hashed_password == user.hashed_password:
# 成功
self.set_secure_cookie("shanbay_user", str(user.id))
self.write_response(user.format_response())
return
# self.render("home.html")
# self.redirect(self.get_argument("next", "/"))
else:
logging.error("incorrect password")
self.render("login.html", error="incorrect password")
return
def write_error(self, status_code, **kwargs):
http_explanations = {
400: 'Request not properly formatted or contains languages that Apertium APy does not support',
404: 'Resource requested does not exist. URL may have been mistyped',
408: 'Server did not receive a complete request within the time it was prepared to wait. Try again',
500: 'Unexpected condition on server. Request could not be fulfilled.',
}
explanation = kwargs.get('explanation', http_explanations.get(status_code, ''))
if 'exc_info' in kwargs and len(kwargs['exc_info']) > 1:
exception = kwargs['exc_info'][1]
if hasattr(exception, 'log_message') and exception.log_message:
explanation = exception.log_message % exception.args
elif hasattr(exception, 'reason'):
explanation = exception.reason or tornado.httputil.responses.get(status_code, 'Unknown')
else:
explanation = tornado.httputil.responses.get(status_code, 'Unknown')
result = {
'status': 'error',
'code': status_code,
'message': tornado.httputil.responses.get(status_code, 'Unknown'),
'explanation': explanation,
}
data = escape.json_encode(result)
self.set_header('Content-Type', 'application/json; charset=UTF-8')
if self.callback:
self.set_header('Content-Type', 'application/javascript; charset=UTF-8')
self._write_buffer.append(utf8('%s(%s)' % (self.callback, data)))
else:
self._write_buffer.append(utf8(data))
self.finish()
def save_blog(blog_url):
print blog_url
req = urllib2.Request(blog_url)
html = urllib2.urlopen(req)
## print html
#html = response.read()
# soup = BeautifulSoup(open("/Users/mio/Desktop/r_blog.html"), "html.parser")
soup = BeautifulSoup(html, "html.parser")
## print soup
# 日期
blog_date = soup.find('span', class_="blogDetail-ownerOther-date")
blog_date = utf8(blog_date.contents[0])
print blog_date
# 标题
title = soup.find('h2', class_="blogDetail-title")
title = utf8(title.contents[0])
title = title.replace("/", "\\")
print title
# print soup
a = soup.find_all("div", class_="blogDetail-content")
blog_content = a[0]
filename = blog_date.replace(':','-')
with open("{}{}.html".format(download_dir, filename), "wb") as fw:
fw.write("# {}\n".format(title))
fw.write("> {}\n\n".format(blog_date))
for i in blog_content:
try:
fw.write(str(i))
except Exception as e:
print e
pass
return get_next(soup)
def save_blog(blog_url):
print blog_url
html = requests.get(blog_url, cookies=cookie).content
# soup = BeautifulSoup(open("/Users/mio/Desktop/r_blog.html"), "html.parser")
soup = BeautifulSoup(html, "html.parser")
# 日期
blog_date = soup.find('span', class_="blogDetail-ownerOther-date")
blog_date = utf8(blog_date.contents[0])
# 标题
title = soup.find('h2', class_="blogDetail-title")
title = utf8(title.contents[0])
title = title.replace("/", "\\")
print title
# print soup
a = soup.find_all("div", class_="blogDetail-content")
blog_content = a[0]
with open("{}{}.md".format(download_dir, title), "wb") as fw:
fw.write("# {}\n".format(title))
fw.write("> {}\n\n".format(blog_date))
for i in blog_content:
try:
fw.write(str(i))
except Exception as e:
print e
pass
return get_next(soup)
def _on_connect(self, parsed):
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
if self.request.request_timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + self.request.request_timeout,
self._on_timeout)
if (self.request.validate_cert and
isinstance(self.stream, SSLIOStream)):
match_hostname(self.stream.socket.getpeercert(),
parsed.hostname)
if (self.request.method not in self._SUPPORTED_METHODS and
not self.request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % self.request.method)
for key in ('network_interface',
'proxy_host', 'proxy_port',
'proxy_username', 'proxy_password'):
if getattr(self.request, key, None):
raise NotImplementedError('%s not supported' % key)
if "Host" not in self.request.headers:
self.request.headers["Host"] = parsed.netloc
username, password = None, None
if parsed.username is not None:
username, password = parsed.username, parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password
if username is not None:
auth = utf8(username) + b(":") + utf8(password)
self.request.headers["Authorization"] = (b("Basic ") +
base64.b64encode(auth))
if self.request.user_agent:
self.request.headers["User-Agent"] = self.request.user_agent
if not self.request.allow_nonstandard_methods:
if self.request.method in ("POST", "PUT"):
assert self.request.body is not None
else:
assert self.request.body is None
if self.request.body is not None:
self.request.headers["Content-Length"] = str(len(
self.request.body))
if (self.request.method == "POST" and
"Content-Type" not in self.request.headers):
self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
if self.request.use_gzip:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = ((parsed.path or '/') +
(('?' + parsed.query) if parsed.query else ''))
request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
req_path))]
for k, v in self.request.headers.get_all():
line = utf8(k) + b(": ") + utf8(v)
if b('\n') in line:
raise ValueError('Newline in header: ' + repr(line))
request_lines.append(line)
self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n"))
if self.request.body is not None:
self.stream.write(self.request.body)
self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)
def test_unrecognised_command(self):
self.connect()
for command in ['', ' ']:
self.stream.write(utf8('%s\r\n' % command))
data = self.read_response()
self.assertEqual(data, utf8('500 Error: bad syntax\r\n'))
self.close()
def any_to_bytes(s):
if isinstance(s, unicode_type):
return utf8(s)
elif isinstance(s, bytes):
return s
return utf8(str(s))
def _valid_cache(value, handler, condition, new_condtion, anonymous, now):
if not options.cache_enabled:
return False
if anonymous and handler.current_user:
return False
if condition:
old_cond = value.get("condition", "") if value else ""
rows = conn.mysql.query(condition)
new_cond = ""
for r in rows:
new_cond += str(r)
# unify to utf8, the string result return by pymongo is unicode
new_cond = utf8(new_cond)
old_cond = utf8(old_cond if old_cond else "")
if old_cond != new_cond:
new_condtion["condition"] = new_cond
return False
if value:
if value["expire"] > now:
return True
else:
return False
else:
return False
def __call__(self, request):
data = {}
response = []
def start_response(status, response_headers, exc_info=None):
data["status"] = status
data["headers"] = response_headers
return response.append
app_response = self.wsgi_application(
WSGIContainer.environ(request), start_response)
response.extend(app_response)
body = b("").join(response)
if hasattr(app_response, "close"):
app_response.close()
if not data: raise Exception("WSGI app did not call start_response")
status_code = int(data["status"].split()[0])
headers = data["headers"]
header_set = set(k.lower() for (k,v) in headers)
body = escape.utf8(body)
if "content-length" not in header_set:
headers.append(("Content-Length", str(len(body))))
if "content-type" not in header_set:
headers.append(("Content-Type", "text/html; charset=UTF-8"))
if "server" not in header_set:
headers.append(("Server", "TornadoServer/%s" % tornado.version))
parts = [escape.utf8("HTTP/1.1 " + data["status"] + "\r\n")]
for key, value in headers:
parts.append(escape.utf8(key) + b(": ") + escape.utf8(value) + b("\r\n"))
parts.append(b("\r\n"))
parts.append(body)
request.write(b("").join(parts))
request.finish()
self._log(status_code, request)
def post(self):
login_email = self.get_argument(app_db_model.DOC_KEY_ACCOUNT_EMAIL)
login_password = escape.utf8(self.get_argument(app_db_model.DOC_KEY_ACCOUNT_PASSWORD))
account = None
try:
account_cursor = self.db.account.find({
app_db_model.DOC_KEY_ACCOUNT_EMAIL: login_email
})
while (yield account_cursor.fetch_next):
account = account_cursor.next_object()
break
except Exception as e:
logger.error(e)
if not account:
self.set_status(404, 'login error')
self.finish()
return
hashed_password = bcrypt.hashpw(login_password, escape.utf8(account[app_db_model.DOC_KEY_ACCOUNT_PASSWORD]))
if hashed_password:
self.set_secure_cookie('user', str(account[app_db_model.DOC_KEY_ID]))
self.set_current_user(
str(account[app_db_model.DOC_KEY_ID]),
account[app_db_model.DOC_KEY_ACCOUNT_NAME],
account[app_db_model.DOC_KEY_ACCOUNT_EMAIL])
self.redirect('/')
else:
self.set_status(404, 'login error')
self.finish()
def test_unicode_literal_expression(self):
# Unicode literals should be usable in templates. Note that this
# test simulates unicode characters appearing directly in the
# template file (with utf8 encoding), i.e. \u escapes would not
# be used in the template file itself.
template = Template(utf8(u'{{ "\u00e9" }}'))
self.assertEqual(template.generate(), utf8(u"\u00e9"))
def _render_parts(self, value, parts=[]):
self._LOGGER.info('rendering parts')
self._LOGGER.debug(value)
if isinstance(value, str) or isinstance(value, unicode):
#parts.append(escape.xhtml_escape(value))
parts.append(value)
elif isinstance(value, int):
parts.append(str(value))
elif isinstance(value, datetime.datetime):
parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
elif isinstance(value, dict):
for name, subvalue in value.iteritems():
if not isinstance(subvalue, list):
subvalue = [subvalue]
for subsubvalue in subvalue:
parts.append('<' + escape.utf8(name) + '>')
self._render_parts(subsubvalue, parts)
parts.append('</' + escape.utf8(name) + '>')
elif value == None:
parts.append("")
else:
self._LOGGER.debug(parts)
self._LOGGER.error("Unknown S3 value type %r", value)
raise Exception("Unknown S3 value type %r", value)
def write_error(self, status_code, **kwargs):
# TODO: Is there a tornado fn to get the full list?
http_messages = {
400: 'Bad Request',
404: 'Not Found',
408: 'Request Timeout',
500: 'Internal Error'
}
result = {
'status': 'error',
'code': status_code,
'message': http_messages.get(status_code, ''),
'explanation': kwargs.get('explanation', '')
}
data = escape.json_encode(result)
self.set_header('Content-Type', 'application/json; charset=UTF-8')
if self.callback:
self.set_header('Content-Type', 'application/javascript; charset=UTF-8')
self._write_buffer.append(utf8('%s(%s)' % (self.callback, data)))
else:
self._write_buffer.append(utf8(data))
self.finish()
def get_authenticated_user(self, callback, http_client=None):
"""Gets the OAuth authorized user and access token.
This method should be called from the handler for your
OAuth callback URL to complete the registration process. We run the
callback with the authenticated user dictionary. This dictionary
will contain an ``access_key`` which can be used to make authorized
requests to this service on behalf of the user. The dictionary will
also contain other fields such as ``name``, depending on the service
used.
"""
future = callback
request_key = escape.utf8(self.get_argument("oauth_token"))
oauth_verifier = self.get_argument("oauth_verifier", None)
request_cookie = self.get_cookie("_oauth_request_token")
if not request_cookie:
future.set_exception(AuthError(
"Missing OAuth request token cookie"))
return
self.clear_cookie("_oauth_request_token")
cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")]
if cookie_key != request_key:
future.set_exception(AuthError(
"Request token does not match cookie"))
return
token = dict(key=cookie_key, secret=cookie_secret)
if oauth_verifier:
token["verifier"] = oauth_verifier
if http_client is None:
http_client = self.get_auth_http_client()
http_client.fetch(self._oauth_access_token_url(token),
self.async_callback(self._on_access_token, callback))
def get(self, path="."):
download = self.get_argument("dl", None)
if download:
cwd = os.getcwd()
try:
os.chdir(utf8(path))
self._download_dir_package(path, download)
finally:
os.chdir(cwd)
return
dirs, files = [], []
path = utf8(path)
for i in os.listdir(path):
p = os.path.join(path, i)
try:
st = os.stat(p)
except OSError:
continue
padding = " " * (30 - len(i))
if os.path.isdir(p):
dirs.append(
(i + "/", padding + "{:>15}".format(st.st_nlink))
)
else:
files.append(
(i, padding + "{:>16}".format(humanize.naturalsize(st.st_size, gnu=True)))
)
lst = sorted(dirs) + sorted(files)
self.render("dir.html", path=path, lst=lst)
def write_headers(self, start_line, headers, chunk=None, callback=None):
"""Implements `.HTTPConnection.write_headers`."""
if self.is_client:
self._request_start_line = start_line
# Client requests with a non-empty body must have either a
# Content-Length or a Transfer-Encoding.
self._chunking_output = (
start_line.method in ('POST', 'PUT', 'PATCH') and
'Content-Length' not in headers and
'Transfer-Encoding' not in headers)
else:
self._response_start_line = start_line
self._chunking_output = (
# TODO: should this use
# self._request_start_line.version or
# start_line.version?
self._request_start_line.version == 'HTTP/1.1' and
# 304 responses have no body (not even a zero-length body), and so
# should not have either Content-Length or Transfer-Encoding.
# headers.
start_line.code != 304 and
# No need to chunk the output if a Content-Length is specified.
'Content-Length' not in headers and
# Applications are discouraged from touching Transfer-Encoding,
# but if they do, leave it alone.
'Transfer-Encoding' not in headers)
# If a 1.0 client asked for keep-alive, add the header.
if (self._request_start_line.version == 'HTTP/1.0' and
(self._request_headers.get('Connection', '').lower()
== 'keep-alive')):
headers['Connection'] = 'Keep-Alive'
if self._chunking_output:
headers['Transfer-Encoding'] = 'chunked'
if (not self.is_client and
(self._request_start_line.method == 'HEAD' or
start_line.code == 304)):
self._expected_content_remaining = 0
elif 'Content-Length' in headers:
self._expected_content_remaining = int(headers['Content-Length'])
else:
self._expected_content_remaining = None
lines = [utf8("%s %s %s" % start_line)]
lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()])
for line in lines:
if b'\n' in line:
raise ValueError('Newline in header: ' + repr(line))
if self.stream.closed():
self._write_future = Future()
self._write_future.set_exception(iostream.StreamClosedError())
else:
if callback is not None:
self._write_callback = stack_context.wrap(callback)
else:
self._write_future = Future()
data = b"\r\n".join(lines) + b"\r\n\r\n"
if chunk:
data += self._format_chunk(chunk)
self._pending_write = self.stream.write(data)
self._pending_write.add_done_callback(self._on_write_complete)
return self._write_future
def get(self):
realm = 'test'
opaque = 'asdf'
# Real implementations would use a random nonce.
nonce = "1234"
auth_header = self.request.headers.get('Authorization', None)
if auth_header is not None:
auth_mode, params = auth_header.split(' ', 1)
assert auth_mode == 'Digest'
param_dict = {}
for pair in params.split(','):
k, v = pair.strip().split('=', 1)
if v[0] == '"' and v[-1] == '"':
v = v[1:-1]
param_dict[k] = v
assert param_dict['realm'] == realm
assert param_dict['opaque'] == opaque
assert param_dict['nonce'] == nonce
assert param_dict['username'] == self.username
assert param_dict['uri'] == self.request.path
h1 = md5(utf8('%s:%s:%s' % (self.username, realm, self.password))).hexdigest()
h2 = md5(utf8('%s:%s' % (self.request.method,
self.request.path))).hexdigest()
digest = md5(utf8('%s:%s:%s' % (h1, nonce, h2))).hexdigest()
if digest == param_dict['response']:
self.write('ok')
else:
self.write('fail')
else:
self.set_status(401)
self.set_header('WWW-Authenticate',
'Digest realm="%s", nonce="%s", opaque="%s"' %
(realm, nonce, opaque))
def prepare_request(cls, request, default_host):
parsed = urlparse.urlsplit(_unicode(request.url))
if request.method not in cls._SUPPORTED_METHODS and not request.allow_nonstandard_methods:
raise KeyError("unknown method %s" % request.method)
request.follow_redirects = False
for key in (
"network_interface",
"proxy_host",
"proxy_port",
"proxy_username",
"proxy_password",
"expect_100_continue",
"body_producer",
):
if getattr(request, key, None):
raise NotImplementedError("%s not supported" % key)
request.headers.pop("Connection", None)
if "Host" not in request.headers:
if not parsed.netloc:
request.headers["Host"] = default_host
elif "@" in parsed.netloc:
request.headers["Host"] = parsed.netloc.rpartition("@")[-1]
else:
request.headers["Host"] = parsed.netloc
username, password = None, None
if parsed.username is not None:
username, password = parsed.username, parsed.password
elif request.auth_username is not None:
username = request.auth_username
password = request.auth_password or ""
if username is not None:
if request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s", request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
request.headers["Authorization"] = b"Basic " + base64.b64encode(auth)
if request.user_agent:
request.headers["User-Agent"] = request.user_agent
if not request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
body_expected = request.method in ("POST", "PATCH", "PUT")
body_present = request.body is not None or request.body_producer is not None
if (body_expected and not body_present) or (body_present and not body_expected):
raise ValueError(
"Body must %sbe None for method %s (unless "
"allow_nonstandard_methods is true)" % ("not " if body_expected else "", request.method)
)
if request.body is not None:
# When body_producer is used the caller is responsible for
# setting Content-Length (or else chunked encoding will be used).
request.headers["Content-Length"] = str(len(request.body))
if request.method == "POST" and "Content-Type" not in request.headers:
request.headers["Content-Type"] = "application/x-www-form-urlencoded"
if request.decompress_response:
request.headers["Accept-Encoding"] = "gzip"
request.url = (parsed.path or "/") + (("?" + parsed.query) if parsed.query else "")
return request
def _oauth10a_signature(
consumer_token: Dict[str, Any],
method: str,
url: str,
parameters: Dict[str, Any] = {},
token: Dict[str, Any] = None,
) -> bytes:
"""Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request.
See http://oauth.net/core/1.0a/#signing_process
"""
parts = urllib.parse.urlparse(url)
scheme, netloc, path = parts[:3]
normalized_url = scheme.lower() + "://" + netloc.lower() + path
base_elems = []
base_elems.append(method.upper())
base_elems.append(normalized_url)
base_elems.append(
"&".join(
"%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items())
)
)
base_string = "&".join(_oauth_escape(e) for e in base_elems)
key_elems = [escape.utf8(urllib.parse.quote(consumer_token["secret"], safe="~"))]
key_elems.append(
escape.utf8(urllib.parse.quote(token["secret"], safe="~") if token else "")
)
key = b"&".join(key_elems)
hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1)
return binascii.b2a_base64(hash.digest())[:-1]
def send_email(from_, to_, subject, body, html=None, attachments=[]):
if isinstance(from_, EmailAddress):
from_ = str(from_)
else:
from_ = utf8(from_)
to = [utf8(t) for t in to_]
if html:
message = MIMEMultipart("alternative")
message.attach(MIMEText(body, "plain"))
message.attach(MIMEText(html, "html"))
else:
message = MIMEText(body)
if attachments:
part = message
message = MIMEMultipart("mixed")
message.attach(part)
for filename, data in attachments:
part = MIMEBase("application", "octet-stream")
part.set_payload(data)
encoders.encode_base64(part)
part.add_header("Content-Disposition", "attachment", filename=filename)
message.attach(part)
message["Date"] = formatdate(time.time())
message["From"] = from_
message["To"] = COMMASPACE.join(to)
message["Subject"] = utf8(subject)
def any_to_bytes(s):
if isinstance(s, str):
return utf8(s)
elif isinstance(s, bytes):
return s
return utf8(str(s))
def publish(self, channel, message, method=None):
"""
Publish message into channel of stream.
"""
method = method or DEFAULT_PUBLISH_METHOD
to_publish = [utf8(channel), utf8(method), utf8(message)]
self.pub_stream.send_multipart(to_publish)
def _on_connect(self):
self._remove_timeout()
if self.request.request_timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + self.request.request_timeout,
stack_context.wrap(self._on_timeout))
if (self.request.method not in self._SUPPORTED_METHODS and
not self.request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % self.request.method)
for key in ('network_interface',
'proxy_host', 'proxy_port',
'proxy_username', 'proxy_password'):
if getattr(self.request, key, None):
raise NotImplementedError('%s not supported' % key)
if "Connection" not in self.request.headers:
self.request.headers["Connection"] = "close"
if "Host" not in self.request.headers:
if '@' in self.parsed.netloc:
self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1]
else:
self.request.headers["Host"] = self.parsed.netloc
username, password = None, None
if self.parsed.username is not None:
username, password = self.parsed.username, self.parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password or ''
if username is not None:
auth = utf8(username) + b":" + utf8(password)
self.request.headers["Authorization"] = (b"Basic " +
base64.b64encode(auth))
if self.request.user_agent:
self.request.headers["User-Agent"] = self.request.user_agent
if not self.request.allow_nonstandard_methods:
if self.request.method in ("POST", "PATCH", "PUT"):
assert self.request.body is not None
else:
assert self.request.body is None
if self.request.body is not None:
self.request.headers["Content-Length"] = str(len(
self.request.body))
if (self.request.method == "POST" and
"Content-Type" not in self.request.headers):
self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
if self.request.use_gzip:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = ((self.parsed.path or '/') +
(('?' + self.parsed.query) if self.parsed.query else ''))
request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
req_path))]
for k, v in self.request.headers.get_all():
line = utf8(k) + b": " + utf8(v)
if b'\n' in line:
raise ValueError('Newline in header: ' + repr(line))
request_lines.append(line)
self.stream.write(b"\r\n".join(request_lines) + b"\r\n\r\n")
if self.request.body is not None:
self.stream.write(self.request.body)
self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)
def make_request_headers(self):
req_path = "/"
request_lines = [utf8("%s %s HTTP/1.1" % (self.method, req_path))]
for k, v in self.headers.items():
line = utf8(k) + b(": ") + utf8(v)
request_lines.append(line)
toreturn = b("\r\n").join(request_lines) + b("\r\n\r\n")
return toreturn
def send_status(self,code,message):
print('status',code,message)
self.code = code
self.message = message
self.status_sent = True
return self.stream.write(b'HTTP/1.1 '+
utf8(denumber(code))+
b' '+utf8(message)+b'\r\n')
def test_not_implemented_command(self):
self.connect()
for command in ['BADCOMMAND', 'unknown', 'Bonzo']:
self.stream.write(utf8('%s\r\n' % command))
data = self.read_response()
self.assertEqual(data, utf8('502 Error: command "%s" not '
'implemented\r\n' % command))
self.close()
def generate_headers(self):
request_lines = [utf8("%s %s HTTP/1.1" % (self.method,
self.uri))]
for k, v in self.headers.items():
line = utf8(k) + b(": ") + utf8(v)
request_lines.append(line)
toreturn = b("\r\n").join(request_lines) + b("\r\n\r\n")
return toreturn
def test_unicode_template(self):
template = Template(utf8(u"\u00e9"))
self.assertEqual(template.generate(), utf8(u"\u00e9"))
def test_apply(self):
def upper(s):
return s.upper()
template = Template(utf8("{% apply upper %}foo{% end %}"))
self.assertEqual(template.generate(upper=upper), b("FOO"))
def test_body_setter(self):
request = HTTPRequest('http://example.com')
request.body = 'foo'
self.assertEqual(request.body, utf8('foo'))
def _finish_with_json(self, callback):
self.log.debug('finishing without templating')
if self.handler._headers.get('Content-Type') is None:
self.handler.set_header('Content-Type',
'application/json; charset=utf-8')
callback(utf8(self.json.to_string()))
def _curl_setup_request(curl, request, buffer, headers):
curl.setopt(pycurl.URL, request.url)
# Request headers may be either a regular dict or HTTPHeaders object
if isinstance(request.headers, httputil.HTTPHeaders):
curl.setopt(pycurl.HTTPHEADER,
[_utf8("%s: %s" % i) for i in request.headers.get_all()])
else:
curl.setopt(pycurl.HTTPHEADER,
[_utf8("%s: %s" % i) for i in request.headers.iteritems()])
if request.header_callback:
curl.setopt(pycurl.HEADERFUNCTION, request.header_callback)
else:
curl.setopt(pycurl.HEADERFUNCTION,
lambda line: _curl_header_callback(headers, line))
if request.streaming_callback:
curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback)
else:
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout))
curl.setopt(pycurl.TIMEOUT, int(request.request_timeout))
if request.user_agent:
curl.setopt(pycurl.USERAGENT, _utf8(request.user_agent))
else:
curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
if request.network_interface:
curl.setopt(pycurl.INTERFACE, request.network_interface)
if request.use_gzip:
curl.setopt(pycurl.ENCODING, "gzip,deflate")
else:
curl.setopt(pycurl.ENCODING, "none")
if request.proxy_host and request.proxy_port:
curl.setopt(pycurl.PROXY, request.proxy_host)
curl.setopt(pycurl.PROXYPORT, request.proxy_port)
if request.proxy_username:
credentials = '%s:%s' % (request.proxy_username,
request.proxy_password)
curl.setopt(pycurl.PROXYUSERPWD, credentials)
else:
curl.setopt(pycurl.PROXY, '')
# Set the request method through curl's retarded interface which makes
# up names for almost every single method
curl_options = {
"GET": pycurl.HTTPGET,
"POST": pycurl.POST,
"PUT": pycurl.UPLOAD,
"HEAD": pycurl.NOBODY,
}
custom_methods = set(["DELETE"])
for o in curl_options.values():
curl.setopt(o, False)
if request.method in curl_options:
curl.unsetopt(pycurl.CUSTOMREQUEST)
curl.setopt(curl_options[request.method], True)
elif request.allow_nonstandard_methods or request.method in custom_methods:
curl.setopt(pycurl.CUSTOMREQUEST, request.method)
else:
raise KeyError('unknown method ' + request.method)
# Handle curl's cryptic options for every individual HTTP method
if request.method in ("POST", "PUT"):
request_buffer = cStringIO.StringIO(escape.utf8(request.body))
curl.setopt(pycurl.READFUNCTION, request_buffer.read)
if request.method == "POST":
def ioctl(cmd):
if cmd == curl.IOCMD_RESTARTREAD:
request_buffer.seek(0)
curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
else:
curl.setopt(pycurl.INFILESIZE, len(request.body))
if request.auth_username and request.auth_password:
userpwd = "%s:%s" % (request.auth_username, request.auth_password)
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
curl.setopt(pycurl.USERPWD, userpwd)
logging.info("%s %s (username: %r)", request.method, request.url,
request.auth_username)
else:
curl.unsetopt(pycurl.USERPWD)
logging.info("%s %s", request.method, request.url)
if threading.activeCount() > 1:
# libcurl/pycurl is not thread-safe by default. When multiple threads
# are used, signals should be disabled. This has the side effect
# of disabling DNS timeouts in some environments (when libcurl is
# not linked against ares), so we don't do it when there is only one
# thread. Applications that use many short-lived threads may need
# to set NOSIGNAL manually in a prepare_curl_callback since
# there may not be any other threads running at the time we call
# threading.activeCount.
curl.setopt(pycurl.NOSIGNAL, 1)
if request.prepare_curl_callback is not None:
request.prepare_curl_callback(curl)
def _on_connect(self, stream):
if self.final_callback is None:
# final_callback is cleared if we've hit our timeout.
stream.close()
return
self.stream = stream
self.stream.set_close_callback(self.on_connection_close)
self._remove_timeout()
if self.final_callback is None:
return
if self.request.request_timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + self.request.request_timeout,
stack_context.wrap(self._on_timeout))
if (self.request.method not in self._SUPPORTED_METHODS
and not self.request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % self.request.method)
for key in ('network_interface', 'proxy_host', 'proxy_port',
'proxy_username', 'proxy_password'):
if getattr(self.request, key, None):
raise NotImplementedError('%s not supported' % key)
if "Connection" not in self.request.headers:
self.request.headers["Connection"] = "close"
if "Host" not in self.request.headers:
if '@' in self.parsed.netloc:
self.request.headers["Host"] = self.parsed.netloc.rpartition(
'@')[-1]
else:
self.request.headers["Host"] = self.parsed.netloc
username, password = None, None
if self.parsed.username is not None:
username, password = self.parsed.username, self.parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password or ''
if username is not None:
if self.request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s",
self.request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
self.request.headers["Authorization"] = (b"Basic " +
base64.b64encode(auth))
if self.request.user_agent:
self.request.headers["User-Agent"] = self.request.user_agent
if not self.request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
body_expected = self.request.method in ("POST", "PATCH", "PUT")
body_present = (self.request.body is not None
or self.request.body_producer is not None)
if ((body_expected and not body_present)
or (body_present and not body_expected)):
raise ValueError(
'Body must %sbe None for method %s (unelss '
'allow_nonstandard_methods is true)' %
('not ' if body_expected else '', self.request.method))
if self.request.expect_100_continue:
self.request.headers["Expect"] = "100-continue"
if self.request.body is not None:
# When body_producer is used the caller is responsible for
# setting Content-Length (or else chunked encoding will be used).
self.request.headers["Content-Length"] = str(len(
self.request.body))
if (self.request.method == "POST"
and "Content-Type" not in self.request.headers):
self.request.headers[
"Content-Type"] = "application/x-www-form-urlencoded"
if self.request.decompress_response:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = ((self.parsed.path or '/') +
(('?' + self.parsed.query) if self.parsed.query else ''))
self.stream.set_nodelay(True)
self.connection = HTTP1Connection(
self.stream, True,
HTTP1ConnectionParameters(
no_keep_alive=True,
max_header_size=self.max_header_size,
decompress=self.request.decompress_response), self._sockaddr)
start_line = httputil.RequestStartLine(self.request.method, req_path,
'HTTP/1.1')
self.connection.write_headers(start_line, self.request.headers)
if self.request.expect_100_continue:
self._read_response()
else:
self._write_body(True)
def test_comment_directive(self):
template = Template(utf8("{% comment blah blah %}foo"))
self.assertEqual(template.generate(), b("foo"))
def write_headers(self, start_line, headers, chunk=None, callback=None):
"""Implements `.HTTPConnection.write_headers`."""
if self.is_client:
self._request_start_line = start_line
# Client requests with a non-empty body must have either a
# Content-Length or a Transfer-Encoding.
# 不检查是否 Http/1.0 是不完备的。
self._chunking_output = (
start_line.method in ('POST', 'PUT', 'PATCH') and
'Content-Length' not in headers and
'Transfer-Encoding' not in headers)
else:
self._response_start_line = start_line
# 对于 HTTP/1.0 ``self._chunking_output=False``，不支持分块传输编码。
self._chunking_output = (
# TODO: should this use
# self._request_start_line.version or
# start_line.version?
self._request_start_line.version == 'HTTP/1.1' and
# 304 responses have no body (not even a zero-length body), and so
# should not have either Content-Length or Transfer-Encoding.
# headers.
start_line.code != 304 and
# No need to chunk the output if a Content-Length is specified.
'Content-Length' not in headers and
# Applications are discouraged from touching Transfer-Encoding,
# but if they do, leave it alone.
'Transfer-Encoding' not in headers)
# If a 1.0 client asked for keep-alive, add the header.
# HTTP/1.1 默认就是持久化连接，不需要单独指定。
# 假设客户端请求使用 HTTP/1.0 和 `Connection:Keep-Alive`，服务端响应时没有指定
# `Content-Length` （比如在 handler 中多次调用 flush 方法）,那么响应数据就无法
# 判断边界，代码中应该对这个条件做特别处理。
if (self._request_start_line.version == 'HTTP/1.0' and
(self._request_headers.get('Connection', '').lower()
== 'keep-alive')):
headers['Connection'] = 'Keep-Alive'
if self._chunking_output:
headers['Transfer-Encoding'] = 'chunked'
# 服务端响应 `HEAD` 或者 304 时不需要 body 数据。
if (not self.is_client and
(self._request_start_line.method == 'HEAD' or
start_line.code == 304)):
self._expected_content_remaining = 0
elif 'Content-Length' in headers:
self._expected_content_remaining = int(headers['Content-Length'])
else:
self._expected_content_remaining = None
lines = [utf8("%s %s %s" % start_line)]
# 通过 add 添加的响应头会输出多个，比如：“Set-Cookie” 响应头。
lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()])
for line in lines:
if b'\n' in line:
raise ValueError('Newline in header: ' + repr(line))
future = None
if self.stream.closed():
future = self._write_future = Future()
future.set_exception(iostream.StreamClosedError())
else:
# "写回调" 是一个实例字段 `_write_callback`，当上一次写操作还没有回调时就再次执行
# 写操作，那么上一次写操作的回调将被放弃（callback is not None）
if callback is not None:
self._write_callback = stack_context.wrap(callback)
else:
# 没有 callback 时，返回 Future(self._write_future)
future = self._write_future = Future()
# Headers
data = b"\r\n".join(lines) + b"\r\n\r\n"
# message-body
if chunk:
data += self._format_chunk(chunk)
self._pending_write = self.stream.write(data)
self._pending_write.add_done_callback(self._on_write_complete)
return future
def __str__(self):
return '%s <%s>' % (utf8(self.name), utf8(self.addr))
def _get_path(self, path, multipart=False):
# If the file doesn't exist, raise a 404: Not Found
try:
path = path.resolve()
except OSError:
raise HTTPError(status_code=NOT_FOUND)
self.path = path
if self.path.is_dir():
self.file = self.path / self.default_filename if self.default_filename else self.path
if not (self.default_filename
and self.file.exists()) and not self.index:
raise HTTPError(status_code=NOT_FOUND)
# Ensure URL has a trailing '/' when displaying the index / default file
if not self.request.path.endswith('/'):
self.redirect(self.request.path + '/', permanent=True)
return
else:
self.file = self.path
if not self.file.exists():
raise HTTPError(status_code=NOT_FOUND)
elif not self.file.is_file():
raise HTTPError(status_code=FORBIDDEN,
log_message='%s is not a file' % self.path)
if not self.allowed(self.file):
raise HTTPError(status_code=FORBIDDEN)
if self.path.is_dir() and self.index and not (self.default_filename
and self.file.exists()):
self.set_header('Content-Type', 'text/html; charset=UTF-8')
content = []
file_template = string.Template(
u'<li><a href="$path">$name</a></li>')
for path in self.path.iterdir():
if path.is_symlink():
name_suffix, path_suffix = ' ►', ''
elif path.is_dir():
name_suffix = path_suffix = '/'
else:
name_suffix = path_suffix = ''
# On Windows, pathlib on Python 2.7 won't handle Unicode. Ignore such files.
# https://bitbucket.org/pitrou/pathlib/issues/25
try:
path = str(path.relative_to(self.path))
content.append(
file_template.substitute(
path=path + path_suffix,
name=path + name_suffix,
))
except UnicodeDecodeError:
app_log.warning(
"FileHandler can't show unicode file {!r:s}".format(
path))
content.append(u'</ul>')
self.content = self.index_template.substitute(
path=self.path, body=''.join(content))
else:
modified = self.file.stat().st_mtime
self.set_header('Last-Modified',
datetime.datetime.utcfromtimestamp(modified))
mime_type = mimetypes.types_map.get(self.file.suffix)
if mime_type is not None:
if mime_type.startswith('text/'):
mime_type += '; charset=UTF-8'
self.set_header('Content-Type', mime_type)
for header_name, header_value in self.headers.items():
if isinstance(header_value, dict):
if _match(self.file, header_name):
for header_name, header_value in header_value.items():
self.set_header(header_name, header_value)
else:
self.set_header(header_name, header_value)
transform = {}
for pattern, trans in self.transform.items():
if _match(self.file, pattern):
transform = trans
break
encoding = transform.get('encoding')
with self.file.open('rb' if encoding is None else 'r',
encoding=encoding) as file:
self.content = file.read()
if transform:
for header_name, header_value in transform[
'headers'].items():
self.set_header(header_name, header_value)
output = []
for item in transform['function'](content=self.content,
handler=self):
if tornado.concurrent.is_future(item):
item = yield item
output.append(item)
self.content = ''.join(output)
self.set_header('Content-Length', len(utf8(self.content)))
if self.include_body:
self.write(self.content)
# Do not flush unless it's multipart. Flushing disables Etag
if multipart:
self.flush()
def _trigger_include_host_check(self, include_host):
path = "/override_static_url/robots.txt?include_host=%s"
response = self.fetch(path % int(include_host))
self.assertEqual(response.body, utf8(str(True)))
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
connect_timeout=20.0,
request_timeout=20.0,
if_modified_since=None,
follow_redirects=True,
max_redirects=5,
user_agent=None,
use_gzip=True,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password='',
allow_nonstandard_methods=False,
validate_cert=True,
ca_certs=None):
if headers is None:
headers = httputil.HTTPHeaders()
if if_modified_since:
timestamp = calendar.timegm(if_modified_since.utctimetuple())
headers["If-Modified-Since"] = email.utils.formatdate(
timestamp, localtime=False, usegmt=True)
# Proxy support: proxy_host and proxy_port must be set to connect via
# proxy. The username and password credentials are optional.
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = utf8(url)
self.method = method
self.headers = headers
self.body = body
self.auth_username = utf8(auth_username)
self.auth_password = utf8(auth_password)
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
# SSL certificate validation:
# validate_cert: boolean, set to False to disable validation
# ca_certs: filename of CA certificates in PEM format, or
# None to use defaults
# Note that in the curl-based HTTP client, if any request
# uses a custom ca_certs file, they all must (they don't have to
# all use the same ca_certs, but it's not possible to mix requests
# with ca_certs and requests that use the defaults).
# SimpleAsyncHTTPClient does not have this limitation.
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.start_time = time.time()
def test_comment(self):
template = Template("Hello{# TODO i18n #} {{ name }}!")
self.assertEqual(template.generate(name=utf8("Ben")), b("Hello Ben!"))
def body(self, value: Union[bytes, str]) -> None:
self._body = utf8(value)
def test_bytes(self):
template = Template("Hello {{ name }}!")
self.assertEqual(template.generate(name=utf8("Ben")), b("Hello Ben!"))
def flush_messages(self):
# type: () -> bool
if self.client_closed or len(self.message_queue) == 0:
return False
msg = ("{\"msgs\":[" + ",".join(self.message_queue) + "]}")
self.message_queue = []
try:
binmsg = utf8(msg)
self.total_message_bytes += len(binmsg)
if self.deflate:
# Compress like in deflate-frame extension:
# Apply deflate, flush, then remove the 00 00 FF FF
# at the end
compressed = self._compressobj.compress(binmsg)
compressed += self._compressobj.flush(zlib.Z_SYNC_FLUSH)
compressed = compressed[:-4]
self.compressed_bytes_sent += len(compressed)
f = self.write_message(compressed, binary=True)
else:
self.uncompressed_bytes_sent += len(binmsg)
f = self.write_message(binmsg)
import traceback
cur_stack = traceback.format_stack()
# handle any exceptions lingering in the Future
# TODO: this whole call chain should be converted to use coroutines
def after_write_callback(f):
try:
f.result()
except tornado.websocket.StreamClosedError as e:
# not supposed to be raised here in current versions of
# tornado, but in some older versions it is.
if self.failed_messages == 0:
self.logger.warning(
"Connection closed during async write_message")
self.failed_messages += 1
if self.ws_connection is not None:
self.ws_connection._abort()
except tornado.websocket.WebSocketClosedError as e:
if self.failed_messages == 0:
self.logger.warning(
"Connection closed during async write_message")
self.failed_messages += 1
if self.ws_connection is not None:
self.ws_connection._abort()
except Exception as e:
self.logger.warning(
"Exception during async write_message, stack at call:")
self.logger.warning("".join(cur_stack))
self.logger.warning(e, exc_info=True)
self.failed_messages += 1
if self.ws_connection is not None:
self.ws_connection._abort()
# extreme back-compat try-except block, `f` should be None in
# ancient tornado versions
try:
f.add_done_callback(after_write_callback)
except:
pass
# true means that something was queued up to send, but it may be
# async
return True
except:
self.logger.warning("Exception trying to send message.",
exc_info=True)
self.failed_messages += 1
if self.ws_connection is not None:
self.ws_connection._abort()
return False
def _on_connect(self):
self._remove_timeout()
if self.request.request_timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + self.request.request_timeout,
stack_context.wrap(self._on_timeout))
if (self.request.method not in self._SUPPORTED_METHODS and
not self.request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % self.request.method)
for key in ('network_interface',
'proxy_host', 'proxy_port',
'proxy_username', 'proxy_password'):
if getattr(self.request, key, None):
raise NotImplementedError('%s not supported' % key)
if "Connection" not in self.request.headers:
self.request.headers["Connection"] = "close"
if "Host" not in self.request.headers:
if '@' in self.parsed.netloc:
self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1]
else:
self.request.headers["Host"] = self.parsed.netloc
username, password = None, None
if self.parsed.username is not None:
username, password = self.parsed.username, self.parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password or ''
if username is not None:
if self.request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s",
self.request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
self.request.headers["Authorization"] = (b"Basic " +
base64.b64encode(auth))
if self.request.user_agent:
self.request.headers["User-Agent"] = self.request.user_agent
if not self.request.allow_nonstandard_methods:
if self.request.method in ("POST", "PATCH", "PUT"):
assert self.request.body is not None
else:
assert self.request.body is None
if self.request.body is not None:
self.request.headers["Content-Length"] = str(len(
self.request.body))
if (self.request.method == "POST" and
"Content-Type" not in self.request.headers):
self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
if self.request.use_gzip:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = ((self.parsed.path or '/') +
(('?' + self.parsed.query) if self.parsed.query else ''))
request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
req_path))]
for k, v in self.request.headers.get_all():
line = utf8(k) + b": " + utf8(v)
if b'\n' in line:
raise ValueError('Newline in header: ' + repr(line))
request_lines.append(line)
request_str = b"\r\n".join(request_lines) + b"\r\n\r\n"
if self.request.body is not None:
request_str += self.request.body
self.stream.set_nodelay(True)
self.stream.write(request_str)
self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)
def body(self, value):
self._body = utf8(value)
def test_absolute_static_url(self):
response = self.fetch("/abs_static_url/robots.txt")
self.assertEqual(response.body,
utf8(self.get_url("/") + "static/robots.txt?v=f71d2"))
def write_response(self):
yield self.stream.write(
utf8("HTTP/1.0 200 OK\r\nContent-Length: %s\r\n\r\nok" %
self.get_argument("value")))
self.stream.close()
def test_unicode_escapes(self):
self.assertEqual(utf8(u'\u00e9'), b'\xc3\xa9')
def _curl_setup_request(curl, request, buffer, headers):
curl.setopt(pycurl.URL, native_str(request.url))
# libcurl's magic "Expect: 100-continue" behavior causes delays
# with servers that don't support it (which include, among others,
# Google's OpenID endpoint). Additionally, this behavior has
# a bug in conjunction with the curl_multi_socket_action API
# (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
# which increases the delays. It's more trouble than it's worth,
# so just turn off the feature (yes, setting Expect: to an empty
# value is the official way to disable this)
if "Expect" not in request.headers:
request.headers["Expect"] = ""
# libcurl adds Pragma: no-cache by default; disable that too
if "Pragma" not in request.headers:
request.headers["Pragma"] = ""
# Request headers may be either a regular dict or HTTPHeaders object
if isinstance(request.headers, httputil.HTTPHeaders):
curl.setopt(
pycurl.HTTPHEADER,
[native_str("%s: %s" % i) for i in request.headers.get_all()])
else:
curl.setopt(
pycurl.HTTPHEADER,
[native_str("%s: %s" % i) for i in request.headers.items()])
if request.header_callback:
curl.setopt(pycurl.HEADERFUNCTION,
lambda line: request.header_callback(native_str(line)))
else:
curl.setopt(
pycurl.HEADERFUNCTION,
lambda line: _curl_header_callback(headers, native_str(line)))
if request.streaming_callback:
write_function = request.streaming_callback
else:
write_function = buffer.write
if bytes_type is str: # py2
curl.setopt(pycurl.WRITEFUNCTION, write_function)
else: # py3
# Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
# a fork/port. That version has a bug in which it passes unicode
# strings instead of bytes to the WRITEFUNCTION. This means that
# if you use a WRITEFUNCTION (which tornado always does), you cannot
# download arbitrary binary data. This needs to be fixed in the
# ported pycurl package, but in the meantime this lambda will
# make it work for downloading (utf8) text.
curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s)))
curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
if request.user_agent:
curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
else:
curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
if request.network_interface:
curl.setopt(pycurl.INTERFACE, request.network_interface)
if request.use_gzip:
curl.setopt(pycurl.ENCODING, "gzip,deflate")
else:
curl.setopt(pycurl.ENCODING, "none")
if request.proxy_host and request.proxy_port:
curl.setopt(pycurl.PROXY, request.proxy_host)
curl.setopt(pycurl.PROXYPORT, request.proxy_port)
if request.proxy_username:
credentials = '%s:%s' % (request.proxy_username,
request.proxy_password)
curl.setopt(pycurl.PROXYUSERPWD, credentials)
else:
curl.setopt(pycurl.PROXY, '')
curl.unsetopt(pycurl.PROXYUSERPWD)
if request.validate_cert:
curl.setopt(pycurl.SSL_VERIFYPEER, 1)
curl.setopt(pycurl.SSL_VERIFYHOST, 2)
else:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
if request.ca_certs is not None:
curl.setopt(pycurl.CAINFO, request.ca_certs)
else:
# There is no way to restore pycurl.CAINFO to its default value
# (Using unsetopt makes it reject all certificates).
# I don't see any way to read the default value from python so it
# can be restored later. We'll have to just leave CAINFO untouched
# if no ca_certs file was specified, and require that if any
# request uses a custom ca_certs file, they all must.
pass
if request.allow_ipv6 is False:
# Curl behaves reasonably when DNS resolution gives an ipv6 address
# that we can't reach, so allow ipv6 unless the user asks to disable.
# (but see version check in _process_queue above)
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
else:
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
# Set the request method through curl's irritating interface which makes
# up names for almost every single method
curl_options = {
"GET": pycurl.HTTPGET,
"POST": pycurl.POST,
"PUT": pycurl.UPLOAD,
"HEAD": pycurl.NOBODY,
}
custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
for o in curl_options.values():
curl.setopt(o, False)
if request.method in curl_options:
curl.unsetopt(pycurl.CUSTOMREQUEST)
curl.setopt(curl_options[request.method], True)
elif request.allow_nonstandard_methods or request.method in custom_methods:
curl.setopt(pycurl.CUSTOMREQUEST, request.method)
else:
raise KeyError('unknown method ' + request.method)
# Handle curl's cryptic options for every individual HTTP method
if request.method in ("POST", "PUT"):
if request.body is None:
raise AssertionError('Body must not be empty for "%s" request' %
request.method)
request_buffer = BytesIO(utf8(request.body))
curl.setopt(pycurl.READFUNCTION, request_buffer.read)
if request.method == "POST":
def ioctl(cmd):
if cmd == curl.IOCMD_RESTARTREAD:
request_buffer.seek(0)
curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
else:
curl.setopt(pycurl.INFILESIZE, len(request.body))
elif request.method == "GET":
if request.body is not None:
raise AssertionError('Body must be empty for GET request')
if request.auth_username is not None:
userpwd = "%s:%s" % (request.auth_username, request.auth_password
or '')
if request.auth_mode is None or request.auth_mode == "basic":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
elif request.auth_mode == "digest":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
else:
raise ValueError("Unsupported auth_mode %s" % request.auth_mode)
curl.setopt(pycurl.USERPWD, native_str(userpwd))
gen_log.debug("%s %s (username: %r)", request.method, request.url,
request.auth_username)
else:
curl.unsetopt(pycurl.USERPWD)
gen_log.debug("%s %s", request.method, request.url)
if request.client_cert is not None:
curl.setopt(pycurl.SSLCERT, request.client_cert)
if request.client_key is not None:
curl.setopt(pycurl.SSLKEY, request.client_key)
if threading.activeCount() > 1:
# libcurl/pycurl is not thread-safe by default. When multiple threads
# are used, signals should be disabled. This has the side effect
# of disabling DNS timeouts in some environments (when libcurl is
# not linked against ares), so we don't do it when there is only one
# thread. Applications that use many short-lived threads may need
# to set NOSIGNAL manually in a prepare_curl_callback since
# there may not be any other threads running at the time we call
# threading.activeCount.
curl.setopt(pycurl.NOSIGNAL, 1)
if request.prepare_curl_callback is not None:
request.prepare_curl_callback(curl)
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
connect_timeout=20.0,
request_timeout=20.0,
if_modified_since=None,
follow_redirects=True,
max_redirects=5,
user_agent=None,
use_gzip=True,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password='',
allow_nonstandard_methods=False,
validate_cert=True,
ca_certs=None,
allow_ipv6=None,
client_key=None,
client_cert=None):
"""Creates an `HTTPRequest`.
All parameters except `url` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP "Basic" authentication
:arg string auth_password: Password for HTTP "Basic" authentication
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg datetime if_modified_since: Timestamp for ``If-Modified-Since``
header
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for `follow_redirects`
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request
:arg callable streaming_callback: If set, `streaming_callback` will
be run with each chunk of data as it is received, and
`~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in
the final response.
:arg callable header_callback: If set, `header_callback` will
be run with each header line as it is received, and
`~HTTPResponse.headers` will be empty in the final response.
:arg callable prepare_curl_callback: If set, will be called with
a `pycurl.Curl` object to allow the application to make additional
`setopt` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
`proxy_host` and `proxy_port` must be set; `proxy_username` and
`proxy_pass` are optional. Proxies are currently only support
with `curl_httpclient`.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for `method`
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. Note that in `curl_httpclient`, if
any request uses a custom `ca_certs` file, they all must (they
don't have to all use the same `ca_certs`, but it's not possible
to mix requests with ca_certs and requests that use the defaults.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
`simple_httpclient` and true in `curl_httpclient`
:arg string client_key: Filename for client SSL key, if any
:arg string client_cert: Filename for client SSL certificate, if any
"""
if headers is None:
headers = httputil.HTTPHeaders()
if if_modified_since:
timestamp = calendar.timegm(if_modified_since.utctimetuple())
headers["If-Modified-Since"] = email.utils.formatdate(
timestamp, localtime=False, usegmt=True)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.headers = headers
self.body = utf8(body)
self.auth_username = auth_username
self.auth_password = auth_password
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
def test_if(self):
template = Template(utf8("{% if x > 4 %}yes{% else %}no{% end %}"))
self.assertEqual(template.generate(x=5), b("yes"))
self.assertEqual(template.generate(x=3), b("no"))
def write_headers(
self,
start_line: Union[httputil.RequestStartLine,
httputil.ResponseStartLine],
headers: httputil.HTTPHeaders,
chunk: Optional[bytes] = None,
) -> "Future[None]":
"""Implements `.HTTPConnection.write_headers`."""
lines = []
if self.is_client:
assert isinstance(start_line, httputil.RequestStartLine)
self._request_start_line = start_line
lines.append(
utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1])))
# Client requests with a non-empty body must have either a
# Content-Length or a Transfer-Encoding.
self._chunking_output = (
start_line.method in ("POST", "PUT", "PATCH")
and "Content-Length" not in headers
and ("Transfer-Encoding" not in headers
or headers["Transfer-Encoding"] == "chunked"))
else:
assert isinstance(start_line, httputil.ResponseStartLine)
assert self._request_start_line is not None
assert self._request_headers is not None
self._response_start_line = start_line
lines.append(
utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2])))
self._chunking_output = (
# TODO: should this use
# self._request_start_line.version or
# start_line.version?
self._request_start_line.version == "HTTP/1.1"
# Omit payload header field for HEAD request.
and self._request_start_line.method != "HEAD"
# 1xx, 204 and 304 responses have no body (not even a zero-length
# body), and so should not have either Content-Length or
# Transfer-Encoding headers.
and start_line.code not in (204, 304)
and (start_line.code < 100 or start_line.code >= 200)
# No need to chunk the output if a Content-Length is specified.
and "Content-Length" not in headers
# Applications are discouraged from touching Transfer-Encoding,
# but if they do, leave it alone.
and "Transfer-Encoding" not in headers)
# If connection to a 1.1 client will be closed, inform client
if (self._request_start_line.version == "HTTP/1.1"
and self._disconnect_on_finish):
headers["Connection"] = "close"
# If a 1.0 client asked for keep-alive, add the header.
if (self._request_start_line.version == "HTTP/1.0"
and self._request_headers.get("Connection",
"").lower() == "keep-alive"):
headers["Connection"] = "Keep-Alive"
if self._chunking_output:
headers["Transfer-Encoding"] = "chunked"
if not self.is_client and (self._request_start_line.method == "HEAD"
or cast(httputil.ResponseStartLine,
start_line).code == 304):
self._expected_content_remaining = 0
elif "Content-Length" in headers:
self._expected_content_remaining = int(headers["Content-Length"])
else:
self._expected_content_remaining = None
# TODO: headers are supposed to be of type str, but we still have some
# cases that let bytes slip through. Remove these native_str calls when those
# are fixed.
header_lines = (native_str(n) + ": " + native_str(v)
for n, v in headers.get_all())
lines.extend(line.encode("latin1") for line in header_lines)
for line in lines:
if b"\n" in line:
raise ValueError("Newline in header: " + repr(line))
future = None
if self.stream.closed():
future = self._write_future = Future()
future.set_exception(iostream.StreamClosedError())
future.exception()
else:
future = self._write_future = Future()
data = b"\r\n".join(lines) + b"\r\n\r\n"
if chunk:
data += self._format_chunk(chunk)
self._pending_write = self.stream.write(data)
future_add_done_callback(self._pending_write,
self._on_write_complete)
return future
def handle_connect(self):
logging.info("handle_connect")
self.stream.write(utf8(self.request_data + "\n"))
self.stream.read_until(b'\n', callback=self.handle_read)
def _curl_setup_request(
self,
curl: pycurl.Curl,
request: HTTPRequest,
buffer: BytesIO,
headers: httputil.HTTPHeaders,
) -> None:
curl.setopt(pycurl.URL, native_str(request.url))
# libcurl's magic "Expect: 100-continue" behavior causes delays
# with servers that don't support it (which include, among others,
# Google's OpenID endpoint). Additionally, this behavior has
# a bug in conjunction with the curl_multi_socket_action API
# (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
# which increases the delays. It's more trouble than it's worth,
# so just turn off the feature (yes, setting Expect: to an empty
# value is the official way to disable this)
if "Expect" not in request.headers:
request.headers["Expect"] = ""
# libcurl adds Pragma: no-cache by default; disable that too
if "Pragma" not in request.headers:
request.headers["Pragma"] = ""
curl.setopt(
pycurl.HTTPHEADER,
[
b"%s: %s"
% (native_str(k).encode("ASCII"), native_str(v).encode("ISO8859-1"))
for k, v in request.headers.get_all()
],
)
curl.setopt(
pycurl.HEADERFUNCTION,
functools.partial(
self._curl_header_callback, headers, request.header_callback
),
)
if request.streaming_callback:
def write_function(b: Union[bytes, bytearray]) -> int:
assert request.streaming_callback is not None
self.io_loop.add_callback(request.streaming_callback, b)
return len(b)
else:
write_function = buffer.write # type: ignore
curl.setopt(pycurl.WRITEFUNCTION, write_function)
curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
assert request.connect_timeout is not None
curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
assert request.request_timeout is not None
curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
if request.user_agent:
curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
else:
curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
if request.network_interface:
curl.setopt(pycurl.INTERFACE, request.network_interface)
if request.decompress_response:
curl.setopt(pycurl.ENCODING, "gzip,deflate")
else:
curl.setopt(pycurl.ENCODING, None)
if request.proxy_host and request.proxy_port:
curl.setopt(pycurl.PROXY, request.proxy_host)
curl.setopt(pycurl.PROXYPORT, request.proxy_port)
if request.proxy_username:
assert request.proxy_password is not None
credentials = httputil.encode_username_password(
request.proxy_username, request.proxy_password
)
curl.setopt(pycurl.PROXYUSERPWD, credentials)
if request.proxy_auth_mode is None or request.proxy_auth_mode == "basic":
curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC)
elif request.proxy_auth_mode == "digest":
curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_DIGEST)
else:
raise ValueError(
"Unsupported proxy_auth_mode %s" % request.proxy_auth_mode
)
else:
try:
curl.unsetopt(pycurl.PROXY)
except TypeError: # not supported, disable proxy
curl.setopt(pycurl.PROXY, "")
curl.unsetopt(pycurl.PROXYUSERPWD)
if request.validate_cert:
curl.setopt(pycurl.SSL_VERIFYPEER, 1)
curl.setopt(pycurl.SSL_VERIFYHOST, 2)
else:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
if request.ca_certs is not None:
curl.setopt(pycurl.CAINFO, request.ca_certs)
else:
# There is no way to restore pycurl.CAINFO to its default value
# (Using unsetopt makes it reject all certificates).
# I don't see any way to read the default value from python so it
# can be restored later. We'll have to just leave CAINFO untouched
# if no ca_certs file was specified, and require that if any
# request uses a custom ca_certs file, they all must.
pass
if request.allow_ipv6 is False:
# Curl behaves reasonably when DNS resolution gives an ipv6 address
# that we can't reach, so allow ipv6 unless the user asks to disable.
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
else:
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
# Set the request method through curl's irritating interface which makes
# up names for almost every single method
curl_options = {
"GET": pycurl.HTTPGET,
"POST": pycurl.POST,
"PUT": pycurl.UPLOAD,
"HEAD": pycurl.NOBODY,
}
custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
for o in curl_options.values():
curl.setopt(o, False)
if request.method in curl_options:
curl.unsetopt(pycurl.CUSTOMREQUEST)
curl.setopt(curl_options[request.method], True)
elif request.allow_nonstandard_methods or request.method in custom_methods:
curl.setopt(pycurl.CUSTOMREQUEST, request.method)
else:
raise KeyError("unknown method " + request.method)
body_expected = request.method in ("POST", "PATCH", "PUT")
body_present = request.body is not None
if not request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
if (body_expected and not body_present) or (
body_present and not body_expected
):
raise ValueError(
"Body must %sbe None for method %s (unless "
"allow_nonstandard_methods is true)"
% ("not " if body_expected else "", request.method)
)
if body_expected or body_present:
if request.method == "GET":
# Even with `allow_nonstandard_methods` we disallow
# GET with a body (because libcurl doesn't allow it
# unless we use CUSTOMREQUEST). While the spec doesn't
# forbid clients from sending a body, it arguably
# disallows the server from doing anything with them.
raise ValueError("Body must be None for GET request")
request_buffer = BytesIO(utf8(request.body or ""))
def ioctl(cmd: int) -> None:
if cmd == curl.IOCMD_RESTARTREAD: # type: ignore
request_buffer.seek(0)
curl.setopt(pycurl.READFUNCTION, request_buffer.read)
curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
if request.method == "POST":
curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or ""))
else:
curl.setopt(pycurl.UPLOAD, True)
curl.setopt(pycurl.INFILESIZE, len(request.body or ""))
if request.auth_username is not None:
assert request.auth_password is not None
if request.auth_mode is None or request.auth_mode == "basic":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
elif request.auth_mode == "digest":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
else:
raise ValueError("Unsupported auth_mode %s" % request.auth_mode)
userpwd = httputil.encode_username_password(
request.auth_username, request.auth_password
)
curl.setopt(pycurl.USERPWD, userpwd)
curl_log.debug(
"%s %s (username: %r)",
request.method,
request.url,
request.auth_username,
)
else:
curl.unsetopt(pycurl.USERPWD)
curl_log.debug("%s %s", request.method, request.url)
if request.client_cert is not None:
curl.setopt(pycurl.SSLCERT, request.client_cert)
if request.client_key is not None:
curl.setopt(pycurl.SSLKEY, request.client_key)
if request.ssl_options is not None:
raise ValueError("ssl_options not supported in curl_httpclient")
if threading.active_count() > 1:
# libcurl/pycurl is not thread-safe by default. When multiple threads
# are used, signals should be disabled. This has the side effect
# of disabling DNS timeouts in some environments (when libcurl is
# not linked against ares), so we don't do it when there is only one
# thread. Applications that use many short-lived threads may need
# to set NOSIGNAL manually in a prepare_curl_callback since
# there may not be any other threads running at the time we call
# threading.activeCount.
curl.setopt(pycurl.NOSIGNAL, 1)
if request.prepare_curl_callback is not None:
request.prepare_curl_callback(curl)
def gzip_encode(s):
with closing(compat.StringIO()) as sio:
with gzip.GzipFile(fileobj=sio, mode='wb') as gz_file:
gz_file.write(escape.utf8(s))
return sio.getvalue()
def test_unicode_logging(self):
self.logger.error(u"\u00e9")
self.assertEqual(self.get_output(), utf8(u"\u00e9"))
def test_bytes_logging(self):
with ignore_bytes_warning():
# This will be "\xe9" on python 2 or "b'\xe9'" on python 3
self.logger.error(b"\xe9")
self.assertEqual(self.get_output(), utf8(repr(b"\xe9")))
