def __after__(self):
try:
v = self._values
# strip / to avoid treating http://example.com and http://example.com/
# as different sites
v["website"] = v["website"].strip("/")
user_id = self._handler.current_user.id
usecase_md = markdown.markdown(escape._unicode(v["usecase"]))
if v.get("site", ""):
stmt = "UPDATE site SET sitename = %s,website = %s,description = %s, "\
"usecase = %s,usecase_md = %s,source_url = %s, logo = %s, "\
"updated = UTC_TIMESTAMP() where uuid_ = %s"
args = (v["sitename"], v["website"], v["desc"], \
v["usecase"], usecase_md, v["source_url"], v["logo"], \
v["site"])
self._handler.db.execute(stmt, *args)
else:
if self._handler.db.get("select * from site where website = %s", v["website"]):
self.add_error("website", "This web site already be registered.")
else:
if (not self._handler.is_staff):
user_projects = self._handler.db.get("select count(*) as c from site where user_id = %s", \
self._handler.current_user.id)
if user_projects is not None and user_projects.c >= 3:
# It will need approve if someone create more than three sites
status = const.Status.PENDING
else:
status = const.Status.UNVERIFIED
else:
status = const.Status.ACTIVE
slug = v["website"].lower().strip()
slug = _domain_prefix_re.sub("", slug)
slug = unicodedata.normalize("NFKD", escape._unicode(slug)).encode("ascii", "ignore")
slug = re.sub(r"[^\w]+", " ", slug)
slug = "-".join(slug.split())
if not slug:
slug = "site"
while True:
e = self._handler.db.get("SELECT * FROM site WHERE slug = %s", slug)
if not e:
break
slug += "-" + uuid.uuid4().hex[0:2]
stmt = "INSERT INTO site (sitename,website,description,usecase,usecase_md,source_url,"\
"user_id,logo,uuid_,created,updated,updated_ss,status_,slug) "\
"VALUES (%s,%s,%s,%s,%s,%s,"\
"%s,%s,%s,UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),%s,%s)"
args = (v["sitename"], v["website"], v["desc"], v["usecase"], usecase_md, v["source_url"], \
user_id, v["logo"], uuid.uuid4().hex, status, slug)
self._handler.db.execute(stmt, *args)
except Exception, e:
logging.error(str(e))
self.add_error("sitename", "Submit project error, please try it later.")
def match_list(self, match_list):
arguments = self.request.arguments
if '__NODELAY__' in arguments:
del arguments['__NODELAY__']
if '__BLACKLIST__' in arguments:
del arguments['__BLACKLIST__']
if '__ASYNCLIST__' in arguments:
del arguments['__ASYNCLIST__']
if '__ASYNC_RESULT__' in arguments:
del arguments['__ASYNC_RESULT__']
if '__ENABLE_DEBUG__' in arguments:
del arguments['__ENABLE_DEBUG__']
if '__CONTENT_TYPE__' in arguments:
del arguments['__CONTENT_TYPE__']
if '__JSONP_CALLBACK_VARNAME__' in arguments:
del arguments['__JSONP_CALLBACK_VARNAME__']
match = "|".join(match_list)
self.logging.info(match)
p = re.compile(r'%s'%(match,),re.I|re.M)
for k in arguments.keys():
arguments_string = _unicode(" ".join(arguments[k]))
arguments_string = re.sub("\r|\n","",arguments_string)
if p.search(arguments_string):
return True
if hasattr(self.request,'body'):
self.logging.info(_unicode(self.request.body))
body = re.sub("\r|\n","",_unicode(self.request.body))
if p.search(body):
return True
return False
def respond_post(self, cmd_name):
stime = time.time()
user = yield objs.User.find_one(
{'login_key': self.get_argument('login', '')})
self.set_header('Content-Type', 'application/json; charset=utf-8')
if not cmd_name:
defer.returnValue(json.dumps(dict(
ok=True,
commands=api_handlers.handlers.keys())))
if cmd_name not in api_handlers.handlers:
defer.returnValue(json.dumps(dict(
ok=False,
desc='unknown command')))
callogtuple = (id(self), cmd_name, user['name'] if user else '')
print "API call %d '%s' by '%s' started." % callogtuple
handler = api_handlers.handlers[cmd_name]
args = dict(
(utf8(k), _unicode(v[0]))
for k, v in self.request.arguments.iteritems())
if 'login' in args:
del args['login']
self.set_header('Cache-Control', 'no-cache')
cors_origin = self.request.headers.get('Origin', None)
if cors_origin:
self.set_header('Access-Control-Allow-Origin', '*')
print "API call %d '%s' by '%s' is CORS from '%s'" % (callogtuple + (cors_origin,))
try:
result = yield handler(ApiRequest(user), **args)
except BnwResponse as br:
print "API call %d '%s' by '%s' failed with exception. %f.s" % (callogtuple + ((time.time() - stime),))
defer.returnValue(json.dumps(dict(
ok=False,
desc=str(br)), ensure_ascii=False))
print "API call %d '%s' by '%s' completed. %f.s" % (callogtuple + ((time.time() - stime),))
defer.returnValue(json.dumps(result, ensure_ascii=False))
def prepare_request(cls, request, default_host):
parsed = urlparse.urlsplit(_unicode(request.url))
if request.method not in cls._SUPPORTED_METHODS and not request.allow_nonstandard_methods:
raise KeyError("unknown method %s" % request.method)
request.follow_redirects = False
for key in (
"network_interface",
"proxy_host",
"proxy_port",
"proxy_username",
"proxy_password",
"expect_100_continue",
"body_producer",
):
if getattr(request, key, None):
raise NotImplementedError("%s not supported" % key)
request.headers.pop("Connection", None)
if "Host" not in request.headers:
if not parsed.netloc:
request.headers["Host"] = default_host
elif "@" in parsed.netloc:
request.headers["Host"] = parsed.netloc.rpartition("@")[-1]
else:
request.headers["Host"] = parsed.netloc
username, password = None, None
if parsed.username is not None:
username, password = parsed.username, parsed.password
elif request.auth_username is not None:
username = request.auth_username
password = request.auth_password or ""
if username is not None:
if request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s", request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
request.headers["Authorization"] = b"Basic " + base64.b64encode(auth)
if request.user_agent:
request.headers["User-Agent"] = request.user_agent
if not request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
body_expected = request.method in ("POST", "PATCH", "PUT")
body_present = request.body is not None or request.body_producer is not None
if (body_expected and not body_present) or (body_present and not body_expected):
raise ValueError(
"Body must %sbe None for method %s (unless "
"allow_nonstandard_methods is true)" % ("not " if body_expected else "", request.method)
)
if request.body is not None:
# When body_producer is used the caller is responsible for
# setting Content-Length (or else chunked encoding will be used).
request.headers["Content-Length"] = str(len(request.body))
if request.method == "POST" and "Content-Type" not in request.headers:
request.headers["Content-Type"] = "application/x-www-form-urlencoded"
if request.decompress_response:
request.headers["Accept-Encoding"] = "gzip"
request.url = (parsed.path or "/") + (("?" + parsed.query) if parsed.query else "")
return request
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size, tcp_client,
max_header_size):
self.start_time = io_loop.time() # 连接开始时间
self.io_loop = io_loop
self.client = client # 请求的client，一般都是同一个
self.request = request # 代理的request
self.release_callback = release_callback # 如果取消了fetch，执行的回调
self.final_callback = final_callback # 如果结束了，执行的回调
self.max_buffer_size = max_buffer_size
self.tcp_client = tcp_client # 一个tcp的连接
self.max_header_size = max_header_size
self.code = None
self.headers = None
self.chunks = []
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
self._sockaddr = None
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if request.allow_ipv6 is False:
af = socket.AF_INET
else:
af = socket.AF_UNSPEC
ssl_options = self._get_ssl_options(self.parsed.scheme)
timeout = min(self.request.connect_timeout, self.request.request_timeout)
# 这里执行添加一个timeout到io_loop，也就是如果请求过时什么的
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
# 这个函数connect好像是没有回调的一个函数，最后一个callback是怎么回事儿
# 这个connect是一个被@gen.coroutine的方法
self.tcp_client.connect(host, port, af=af,
ssl_options=ssl_options,
callback=self._on_connect)
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size, tcp_client,
max_header_size, max_body_size):
self.start_time = io_loop.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.max_buffer_size = max_buffer_size
self.tcp_client = tcp_client
self.max_header_size = max_header_size
self.max_body_size = max_body_size
self.code = None
self.headers = None
self.chunks = []
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
self._sockaddr = None
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
af = socket.AF_UNSPEC
netloc = self.parsed.netloc
host, port = httputil.split_host_and_port(netloc)
timeout = min(self.request.connect_timeout, self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.tcp_client.connect(host,port, af=af,
max_buffer_size=self.max_buffer_size,
callback=self._on_connect)
def __init__(self, io_loop, client, request, callback):
self.start_time = time.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.callback = callback
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.StackContext(self.cleanup):
parsed = urlparse.urlsplit(_unicode(self.request.url))
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r"^(.+):(\d+)$", netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if parsed.scheme == "https" else 80
if re.match(r"^\[.*\]$", host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
if request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, 0, 0)
af, socktype, proto, canonname, sockaddr = addrinfo[0]
if parsed.scheme == "https":
ssl_options = {}
if request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if request.ca_certs is not None:
ssl_options["ca_certs"] = request.ca_certs
else:
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
self.stream = SSLIOStream(
socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options
)
else:
self.stream = IOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop)
timeout = min(request.connect_timeout, request.request_timeout)
if timeout:
self._connect_timeout = self.io_loop.add_timeout(self.start_time + timeout, self._on_timeout)
self.stream.set_close_callback(self._on_close)
self.stream.connect(sockaddr, functools.partial(self._on_connect, parsed))
def __call__(self, *fields, **kwargs):
_params = []
_cols = []
if len(fields)<1 and not kwargs:
raise database.OperationalError("Update need some date")
# raise database.OperationalError, 'Update Need some data'
elif kwargs:
for k in kwargs.keys():
if type(kwargs[k]).__name__ == 'unicode':
cond = eval(''.join(['self._queryer.', k, "==u'''", kwargs[k].replace("'''", "\\'\\'\\'"), "'''"]))
elif type(kwargs[k]).__name__ == 'int' or type(kwargs[k]).__name__ == 'long':
cond = eval(''.join(['self._queryer.', k, "==u'''", str(kwargs[k]), "'''"]))
elif type(kwargs[k]).__name__ == 'datetime':
cond = eval(''.join(['self._queryer.', k, "==u'", unicode(kwargs[k]), "'"]))
else:
cond = eval(''.join(['self._queryer.', k, "==u'", _unicode(kwargs[k]), "'"]))
_cols.append(cond.get_sql())
_params.append(cond.get_params()[0])
else:
for f in fields:
_cols.append(f.get_sql())
_params.append(f.get_params()[0])
_sql_slice = ['UPDATE `', self._table, '` SET ', ','.join(_cols)]
if self._where:
_sql_slice.append(' WHERE ')
_sql_slice.append(self._where.get_sql())
for p in self._where.get_params():
_params.append(p)
_sql = ''.join(_sql_slice)
return self._db.execute(_sql,*_params)
raise database.OperationalError("Update need where")
def decode_argument(value, name=None):
"""Decodes an argument from the request.
"""
try:
return _unicode(value)
except UnicodeDecodeError:
raise HTTPError(400, "Invalid unicode in %s: %r" %
(name or "url", value[:40]))
def make_content(text, extra_params='rel="nofollow"'):
"""https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238
"""
if extra_params:
extra_params = " " + extra_params.strip()
def make_link(m):
url = m.group(1)
proto = m.group(2)
href = m.group(1)
if not proto:
href = "http://" + href
params = extra_params
if '.' in href:
name_extension = href.split('.')[-1].lower()
if name_extension in ('jpg', 'png', 'gif', 'jpeg'):
return u'<img src="%s" />' % href
return u'<a href="%s"%s>%s</a>' % (href, params, url)
def cover_email(m):
data = {'mail': m.group(1),
'end': m.group(2)}
return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data
def convert_mention(m):
data = {}
data['begin'], data['user'] = m.group(1).split('@')
t = u'%(begin)s<a href="/member/%(user)s" class="mention">' \
u'@%(user)s</a>'
return t % data
def convert_floor(m):
data = {}
data['begin'], data['floor'] = m.group(1).split('#')
t = u'%(begin)s<a href="#reply%(floor)s"' \
' class="mention mention_floor">#%(floor)s</a>'
return t % data
def convert_topic(m):
data = {}
data['begin'], data['topic_link'] = m.group(1).split('t')
data['topic_link_short'] = data['topic_link'][:6]
t = u"""%(begin)s<a href="%(topic_link)s"
class="mention mention_topic"
_id=%(topic_link)s>t%(topic_link_short)s</a>"""
return t % data
text = _unicode(text).replace('\n', '<br />')
text = text.replace(''', '\'')
text = _EMAIL_RE.sub(cover_email, text)
text = _MENTION_RE.sub(convert_mention, text)
text = _FLOOR_RE.sub(convert_floor, text)
text = _TOPIC_RE.sub(convert_topic, text)
return _URL_RE.sub(make_link, text)
def post(self):
self.finish(
{
"header": self.request.headers["X-Header-Encoding-Test"],
"argument": self.get_argument("argument"),
"filename": self.request.files["files"][0].filename,
"filebody": _unicode(self.request.files["files"][0]["body"]),
}
)
def __init__(self, **kwargs):
if 'form' == self.__datatype__:
self._raw_data = {}
# Processing on handler.request.arguments, utf-8 values
for k in kwargs:
if isinstance(kwargs[k], list):
if len(kwargs[k]) > 1:
self._raw_data[k] = map(_unicode, kwargs[k])
else:
self._raw_data[k] = _unicode(kwargs[k][0])
else:
self._raw_data[k] = _unicode(kwargs[k])
else:
self._raw_data = kwargs
self.data = {}
self.errors = []
self.validate()
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size, resolver):
self.start_time = io_loop.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.max_buffer_size = max_buffer_size
self.resolver = resolver
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
# 支持request的timeout
timeout = min(self.request.connect_timeout, self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
# dns解析模块，解析ip地址
# Tornado 本身提供两个dns解析库，一个是基于多线程的，一个就是直接同步的blocking版本
self.resolver.resolve(host, port, af, callback=self._on_resolve)
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size, tcp_client,
max_header_size, max_body_size):
self.start_time = io_loop.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.max_buffer_size = max_buffer_size
self.tcp_client = tcp_client
self.max_header_size = max_header_size
self.max_body_size = max_body_size
self.code = None
self.headers = None
self.chunks = []
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
self._sockaddr = None
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
host, port = httputil.split_host_and_port(netloc)
if port is None:
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if request.allow_ipv6 is False:
af = socket.AF_INET
else:
af = socket.AF_UNSPEC
ssl_options = self._get_ssl_options(self.parsed.scheme)
timeout = min(self.request.connect_timeout, self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.tcp_client.connect(host, port, af=af,
ssl_options=ssl_options,
max_buffer_size=self.max_buffer_size,
callback=self._on_connect)
def getlist(self, name, strip=True):
values = []
for v in self.get(name, []):
v = _unicode(v)
if isinstance(v, unicode):
# Get rid of any weird control chars (unless decoding gave
# us bytes, in which case leave it alone)
v = re.sub(r"[\x00-\x08\x0e-\x1f]", " ", v)
if strip:
v = v.strip()
values.append(v)
return values
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size):
self.start_time = io_loop.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.max_buffer_size = max_buffer_size
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if ssl is None and self.parsed.scheme == "https":
raise ValueError("HTTPS requires either python2.6+ or "
"curl_httpclient")
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
if request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
self.client.resolver.getaddrinfo(
host, port, af, socket.SOCK_STREAM, 0, 0,
callback=self._on_resolve)
def __init__(self, **kwargs):
self._raw_data = {}
# handler.request.arguments, utf-8 values
for k in kwargs:
if isinstance(kwargs[k], list):
self._raw_data[k] = map(_unicode, kwargs[k])
else:
self._raw_data[k] = _unicode(kwargs[k])
self.data = {}
self.errors = []
self.validate()
def make_content(text, extra_params='rel="nofollow"'):
"""https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238
"""
if extra_params:
extra_params = " " + extra_params.strip()
def make_link(m):
url = m.group(1)
proto = m.group(2)
href = m.group(1)
if not proto:
href = "http://" + href # no proto specified, use http
params = extra_params
if '.' in href:
name_extension = href.split('.')[-1].lower()
if name_extension in ('jpg', 'png', 'git', 'jpeg'):
return u'<img src="%s" />' % href
return u'<a href="%s"%s>%s</a>' % (href, params, url)
def cover_email(m):
data = {'mail': m.group(1),
'end': m.group(2)}
return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data
def convert_mention(m):
data = {}
data['begin'], data['user'] = m.group(1).split('@')
t = u'%(begin)s<a href="/member/%(user)s" class="mention">' \
u'@%(user)s</a>'
return t % data
def highligt(m):
try:
name = m.group(1)
lexer = get_lexer_by_name(name)
except ValueError:
lexer = TextLexer()
text = m.group(2).replace('"', '"').replace('&', '&')
text = text.replace('<', '<').replace('>', '>')
text = text.replace(' ', ' ')
return highlight(text, lexer, formatter)
text = _unicode(xhtml_escape(text)).replace(' ', ' ')
text = _CODE_RE.sub(highligt, text).replace('\n', '<br />')
text = _EMAIL_RE.sub(cover_email, text)
text = _MENTION_RE.sub(convert_mention, text)
return _URL_RE.sub(make_link, text)
def match_list(self, match_list):
arguments = self.request.arguments
if '__NODELAY__' in arguments:
del arguments['__NODELAY__']
if '__BLACKLIST__' in arguments:
del arguments['__BLACKLIST__']
if '__ASYNCLIST__' in arguments:
del arguments['__ASYNCLIST__']
if '__ASYNC_RESULT__' in arguments:
del arguments['__ASYNC_RESULT__']
if '__ENABLE_DEBUG__' in arguments:
del arguments['__ENABLE_DEBUG__']
if '__CONTENT_TYPE__' in arguments:
del arguments['__CONTENT_TYPE__']
if '__JSONP_CALLBACK_VARNAME__' in arguments:
del arguments['__JSONP_CALLBACK_VARNAME__']
match = "|".join(match_list)
for k in arguments.keys():
if re.match(match,_unicode(" ".join(arguments[k]))):
return True
if server_request.body != '' and re.match(match,_unicode(server_request.body)):
return True
return False
def make_content(text, extra_params='rel="nofollow"'):
"""https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238
"""
if extra_params:
extra_params = " " + extra_params.strip()
def make_link(m):
url = m.group(1)
proto = m.group(2)
href = m.group(1)
if not proto:
href = "http://" + href # no proto specified, use http
params = extra_params
if "." in href:
name_extension = href.split(".")[-1]
if name_extension in ("jpg", "png", "git", "jpeg"):
return u'<img src="%s" />' % href
return u'<a href="%s"%s>%s</a>' % (href, params, url)
def cover_email(m):
data = {"mail": m.group(1), "end": m.group(2)}
return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data
def convert_mention(m):
data = {}
data["begin"], data["user"] = m.group(1).split("@")
t = u'%(begin)s<a href="/member/%(user)s" class="mention">' u"@%(user)s</a>"
return t % data
def highligt(m):
try:
name = m.group(1)
lexer = get_lexer_by_name(name)
except ValueError:
lexer = TextLexer()
text = m.group(2).replace(""", '"').replace("&", "&")
text = text.replace("<", "<").replace(">", ">")
return highlight(text, lexer, formatter)
text = _unicode(xhtml_escape(text))
text = _CODE_RE.sub(highligt, text).replace("\n", "<br />")
text = _EMAIL_RE.sub(cover_email, text)
text = _MENTION_RE.sub(convert_mention, text)
return _URL_RE.sub(make_link, text)
def resolve(self):
self.proxy_host = getattr(self.request, 'proxy_host', None)
self.proxy_port = getattr(self.request, 'proxy_port', 8080)
if self.proxy_host:
host = self.proxy_host
port = self.proxy_port
self.is_support_keepalive = False
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if self.request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
self.update_timeout()
if self.is_connected():
self.logger.info('is connecting ....')
self._on_connect()
else:
if not self.proxy_host:
self.resolver.resolve(host, port, af, callback=self._on_resolve)
else:
self.resolver.resolve(self.proxy_host, self.proxy_port, af, callback=self._on_resolve)
def test_url_fragment(self):
# Even though [RFC 3986](http://tools.ietf.org/html/rfc3986#section-3.5) specifically state that:
# "...the fragment identifier is not used in the scheme-specific processing of a URI; instead,
# the fragment identifier is separated from the rest of the URI prior to a dereference, and thus
# the identifying information within the fragment itself is dereferenced solely by the user agent,
# regardless of the URI scheme."
# It is somewhat dubious on whether a web framework like tornado should ignore the fragment.
# Users might need it to decide on what to serve. Consider this other part of the spec:
# "As with any URI, use of a fragment identifier component does not imply that a retrieval action
# will take place. A URI with a fragment identifier may be used to refer to the secondary resource
# without any implication that the primary resource is accessible or will ever be accessed."
# All of this considered, this change only allows users to use fragments if they do see fit.
response = self.fetch("/echofrag#frag=123")
self.assertEqual(200, response.code)
frag, sep, query = _unicode(response.body).partition('\n')
self.assertEqual("frag=123", frag)
def getlist(self, key):
"""
tornado.web.RequestHandler._get_arguments
"""
try:
_remove_control_chars_regex = re.compile(r"[\x00-\x08\x0e-\x1f]")
values = []
for v in self.get(key, []):
if isinstance(v, bytes):
v = _unicode(v)
if isinstance(v, unicode_type):
v = _remove_control_chars_regex.sub(" ", v)
values.append(v)
return values
except UnicodeDecodeError:
return []
def __init__(self, io_loop, client, request, callback):
self.start_time = time.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.callback = callback
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.StackContext(self.cleanup):
parsed = urlparse.urlsplit(_unicode(self.request.url))
host = parsed.hostname
if parsed.port is None:
port = 443 if parsed.scheme == "https" else 80
else:
port = parsed.port
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
if parsed.scheme == "https":
ssl_options = {}
if request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if request.ca_certs is not None:
ssl_options["ca_certs"] = request.ca_certs
else:
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
self.stream = SSLIOStream(socket.socket(),
io_loop=self.io_loop,
ssl_options=ssl_options)
else:
self.stream = IOStream(socket.socket(),
io_loop=self.io_loop)
timeout = min(request.connect_timeout, request.request_timeout)
if timeout:
self._connect_timeout = self.io_loop.add_timeout(
self.start_time + timeout,
self._on_timeout)
self.stream.set_close_callback(self._on_close)
self.stream.connect((host, port),
functools.partial(self._on_connect, parsed))
def format(self, record):
try:
record.message = record.getMessage()
except Exception as e:
record.message = "Bad message (%r): %r" % (e, record.__dict__)
assert isinstance(record.message, basestring_type) # guaranteed by logging
record.asctime = time.strftime(
"%y%m%d %H:%M:%S", self.converter(record.created))
prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \
record.__dict__
if self._color:
prefix = (self._colors.get(record.levelno, self._normal) +
prefix + self._normal)
# Encoding notes: The logging module prefers to work with character
# strings, but only enforces that log messages are instances of
# basestring. In python 2, non-ascii bytestrings will make
# their way through the logging framework until they blow up with
# an unhelpful decoding error (with this formatter it happens
# when we attach the prefix, but there are other opportunities for
# exceptions further along in the framework).
#
# If a byte string makes it this far, convert it to unicode to
# ensure it will make it out to the logs. Use repr() as a fallback
# to ensure that all byte strings can be converted successfully,
# but don't do it by default so we don't add extra quotes to ascii
# bytestrings. This is a bit of a hacky place to do this, but
# it's worth it since the encoding errors that would otherwise
# result are so useless (and tornado is fond of using utf8-encoded
# byte strings whereever possible).
try:
message = _unicode(record.message)
except UnicodeDecodeError:
message = repr(record.message)
formatted = prefix + " " + message
if record.exc_info:
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
formatted = formatted.rstrip() + "\n" + record.exc_text
return formatted.replace("\n", "\n ")
def decode_argument(self, value, name=None):
"""Decodes an argument from the command parameters.
The argument has been percent-decoded and is now a byte string.
By default, this method decodes the argument as utf-8 and returns
a unicode string, but this may be overridden in subclasses.
This method is used as a filter for both `get_argument()` and for
values extracted from the parameters
The name of the argument is provided if known, but may be None
(e.g. for unnamed groups in the url regex).
"""
if not isinstance(value, (unicode_type,)):
return value
try:
return _unicode(value)
except UnicodeDecodeError:
raise CommandError("Invalid unicode in %s: %r" %
(name or "", value[:40]))
def format(self, text, secure=False):
#def linkshit_format(text, secure=False, parser=moin_parser):
text = _unicode(xhtml_escape(text))
out = []
thumbs = []
stack = []
instrong = False
inemph = False
self.formatting_tags.keys()
for m in self.parser.parse(text):
if isinstance(m, tuple):
if m[0] in self._formatfuncs:
new_html, new_thumb = self._formatfuncs[m[0]](m, secure)
if new_html:
out.append(new_html)
if new_thumb:
thumbs.append(new_thumb)
elif m[0] in self.formatting_tags:
tag = self.formatting_tags[m[0]]
if not m[0] in stack:
out.append(tag[0])
stack.insert(0, m[0])
else:
tp = stack.index(m[0])
for uptag in stack[:tp + 1]:
out.append(self.formatting_tags[uptag][1])
for uptag in reversed(stack[:tp]):
out.append(self.formatting_tags[uptag][0])
del stack[tp]
else:
out.append('%s<!-- %s -->' % (m[1], m[0]))
else:
out.append(m)
for i in stack:
out.append(self.formatting_tags[i][1])
return '<div class="pwls">'+''.join(out)+'</div>', thumbs
def _parse_string(self, value):
return _unicode(value)
def decode_argument(self, value):
return _unicode(value)
async def run(self) -> None:
try:
self.parsed = urllib.parse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" % self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
host, port = httputil.split_host_and_port(netloc)
if port is None:
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r"^\[.*\]$", host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if self.request.allow_ipv6 is False:
af = socket.AF_INET
else:
af = socket.AF_UNSPEC
ssl_options = self._get_ssl_options(self.parsed.scheme)
source_ip = None
if self.request.network_interface:
if is_valid_ip(self.request.network_interface):
source_ip = self.request.network_interface
else:
raise ValueError(
"Unrecognized IPv4 or IPv6 address for network_interface, got %r"
% (self.request.network_interface,)
)
timeout = (
min(self.request.connect_timeout, self.request.request_timeout)
or self.request.connect_timeout
or self.request.request_timeout
) # min but skip zero
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
functools.partial(self._on_timeout, "while connecting"),
)
stream = await self.tcp_client.connect(
host,
port,
af=af,
ssl_options=ssl_options,
max_buffer_size=self.max_buffer_size,
source_ip=source_ip,
)
if self.final_callback is None:
# final_callback is cleared if we've hit our timeout.
stream.close()
return
self.stream = stream
self.stream.set_close_callback(self.on_connection_close)
self._remove_timeout()
if self.final_callback is None:
return
if self.request.request_timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + self.request.request_timeout,
functools.partial(self._on_timeout, "during request"),
)
if (
self.request.method not in self._SUPPORTED_METHODS
and not self.request.allow_nonstandard_methods
):
raise KeyError("unknown method %s" % self.request.method)
for key in (
"proxy_host",
"proxy_port",
"proxy_username",
"proxy_password",
"proxy_auth_mode",
):
if getattr(self.request, key, None):
raise NotImplementedError("%s not supported" % key)
if "Connection" not in self.request.headers:
self.request.headers["Connection"] = "close"
if "Host" not in self.request.headers:
if "@" in self.parsed.netloc:
self.request.headers["Host"] = self.parsed.netloc.rpartition("@")[
-1
]
else:
self.request.headers["Host"] = self.parsed.netloc
username, password = None, None
if self.parsed.username is not None:
username, password = self.parsed.username, self.parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password or ""
if username is not None:
assert password is not None
if self.request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s", self.request.auth_mode)
self.request.headers["Authorization"] = "Basic " + _unicode(
base64.b64encode(
httputil.encode_username_password(username, password)
)
)
if self.request.user_agent:
self.request.headers["User-Agent"] = self.request.user_agent
elif self.request.headers.get("User-Agent") is None:
self.request.headers["User-Agent"] = "Tornado/{}".format(version)
if not self.request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
body_expected = self.request.method in ("POST", "PATCH", "PUT")
body_present = (
self.request.body is not None
or self.request.body_producer is not None
)
if (body_expected and not body_present) or (
body_present and not body_expected
):
raise ValueError(
"Body must %sbe None for method %s (unless "
"allow_nonstandard_methods is true)"
% ("not " if body_expected else "", self.request.method)
)
if self.request.expect_100_continue:
self.request.headers["Expect"] = "100-continue"
if self.request.body is not None:
# When body_producer is used the caller is responsible for
# setting Content-Length (or else chunked encoding will be used).
self.request.headers["Content-Length"] = str(len(self.request.body))
if (
self.request.method == "POST"
and "Content-Type" not in self.request.headers
):
self.request.headers[
"Content-Type"
] = "application/x-www-form-urlencoded"
if self.request.decompress_response:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = (self.parsed.path or "/") + (
("?" + self.parsed.query) if self.parsed.query else ""
)
self.connection = self._create_connection(stream)
start_line = httputil.RequestStartLine(self.request.method, req_path, "")
self.connection.write_headers(start_line, self.request.headers)
if self.request.expect_100_continue:
await self.connection.read_response(self)
else:
await self._write_body(True)
except Exception:
if not self._handle_exception(*sys.exc_info()):
raise
def safe_unicode(s):
try:
return _unicode(s)
except UnicodeDecodeError:
return repr(s)
def make_content(text, extra_params='rel="nofollow"'):
"""https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238
"""
if extra_params:
extra_params = " " + extra_params.strip()
def make_link(m):
url = m.group(1)
proto = m.group(2)
href = m.group(1)
if not proto:
href = "http://" + href # no proto specified, use http
params = extra_params
if '.' in href:
name_extension = href.split('.')[-1].lower()
if name_extension in ('jpg', 'png', 'git', 'jpeg'):
return u'<img src="%s" />' % href
return u'<a href="%s"%s>%s</a>' % (href, params, url)
def cover_email(m):
data = {'mail': m.group(1), 'end': m.group(2)}
return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data
def convert_mention(m):
data = {}
data['begin'], data['user'] = m.group(1).split('@')
t = u'%(begin)s<a href="/member/%(user)s" class="mention">' \
u'@%(user)s</a>'
return t % data
def convert_floor(m):
data = {}
data['begin'], data['floor'] = m.group(1).split('#')
t = u'%(begin)s<a href="#reply%(floor)s"' \
' class="mention mention_floor">#%(floor)s</a>'
return t % data
def convert_topic(m):
data = {}
data['begin'], data['topic_link'] = m.group(1).split('t')
data['topic_link_short'] = data['topic_link'][:6]
t = u"""%(begin)s<a href="%(topic_link)s"
class="mention mention_topic"
_id=%(topic_link)s>t%(topic_link_short)s</a>"""
return t % data
def highligt(m):
try:
name = m.group(1)
lexer = get_lexer_by_name(name)
except ValueError:
lexer = TextLexer()
text = m.group(2).replace('"', '"').replace('&', '&')
text = text.replace('<', '<').replace('>', '>')
text = text.replace(' ', ' ')
return highlight(text, lexer, formatter)
text = _unicode(xhtml_escape(text)).replace(' ', ' ')
text = _CODE_RE.sub(highligt, text).replace('\n', '<br />')
text = _EMAIL_RE.sub(cover_email, text)
text = _MENTION_RE.sub(convert_mention, text)
text = _FLOOR_RE.sub(convert_floor, text)
text = _TOPIC_RE.sub(convert_topic, text)
return _URL_RE.sub(make_link, text)
def __after__(self):
try:
v = self._values
# strip / to avoid treating http://example.com and http://example.com/
# as different sites
v["website"] = v["website"].strip("/")
user_id = self._handler.current_user.id
usecase_md = markdown.markdown(escape._unicode(v["usecase"]))
if v.get("site", ""):
stmt = "UPDATE site SET sitename = %s,website = %s,description = %s, "\
"usecase = %s,usecase_md = %s,source_url = %s, logo = %s, "\
"updated = UTC_TIMESTAMP() where uuid_ = %s"
args = (v["sitename"], v["website"], v["desc"], \
v["usecase"], usecase_md, v["source_url"], v["logo"], \
v["site"])
self._handler.db.execute(stmt, *args)
else:
if self._handler.db.get(
"select * from site where website = %s", v["website"]):
self.add_error("website",
"This web site already be registered.")
else:
if (not self._handler.is_staff):
user_projects = self._handler.db.get("select count(*) as c from site where user_id = %s", \
self._handler.current_user.id)
if user_projects is not None and user_projects.c >= 3:
# It will need approve if someone create more than three sites
status = const.Status.PENDING
else:
status = const.Status.UNVERIFIED
else:
status = const.Status.ACTIVE
slug = v["website"].lower().strip()
slug = _domain_prefix_re.sub("", slug)
slug = unicodedata.normalize("NFKD",
escape._unicode(slug)).encode(
"ascii", "ignore")
slug = re.sub(r"[^\w]+", " ", slug)
slug = "-".join(slug.split())
if not slug:
slug = "site"
while True:
e = self._handler.db.get(
"SELECT * FROM site WHERE slug = %s", slug)
if not e:
break
slug += "-" + uuid.uuid4().hex[0:2]
stmt = "INSERT INTO site (sitename,website,description,usecase,usecase_md,source_url,"\
"user_id,logo,uuid_,created,updated,updated_ss,status_,slug) "\
"VALUES (%s,%s,%s,%s,%s,%s,"\
"%s,%s,%s,UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),%s,%s)"
args = (v["sitename"], v["website"], v["desc"], v["usecase"], usecase_md, v["source_url"], \
user_id, v["logo"], uuid.uuid4().hex, status, slug)
self._handler.db.execute(stmt, *args)
except Exception, e:
logging.error(str(e))
self.add_error("sitename",
"Submit project error, please try it later.")
def _safe_unicode(s):
try:
return _unicode(s)
except UnicodeDecodeError:
return repr(s)
def _parse_string(self, value: str) -> str:
return _unicode(value)
def _safe_unicode(s: Any) -> str:
try:
return _unicode(s)
except UnicodeDecodeError:
return repr(s)
def post(self):
self.finish({"header": self.request.headers["X-Header-Encoding-Test"],
"argument": self.get_argument("argument"),
"filename": self.request.files["files"][0].filename,
"filebody": _unicode(self.request.files["files"][0]["body"]),
})
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size):
self.start_time = time.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.StackContext(self.cleanup):
parsed = urllib.parse.urlsplit(_unicode(self.request.url))
if ssl is None and parsed.scheme == "https":
raise ValueError("HTTPS requires either python2.6+ or "
"curl_httpclient")
if parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
parsed_hostname = host # save final parsed host for _on_connect
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
if request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM,
0, 0)
af, socktype, proto, canonname, sockaddr = addrinfo[0]
if parsed.scheme == "https":
ssl_options = {}
if request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if request.ca_certs is not None:
ssl_options["ca_certs"] = request.ca_certs
else:
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
if request.client_key is not None:
ssl_options["keyfile"] = request.client_key
if request.client_cert is not None:
ssl_options["certfile"] = request.client_cert
# SSL interoperability is tricky. We want to disable
# SSLv2 for security reasons; it wasn't disabled by default
# until openssl 1.0. The best way to do this is to use
# the SSL_OP_NO_SSLv2, but that wasn't exposed to python
# until 3.2. Python 2.7 adds the ciphers argument, which
# can also be used to disable SSLv2. As a last resort
# on python 2.6, we set ssl_version to SSLv3. This is
# more narrow than we'd like since it also breaks
# compatibility with servers configured for TLSv1 only,
# but nearly all servers support SSLv3:
# http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
if sys.version_info >= (2, 7):
ssl_options["ciphers"] = "DEFAULT:!SSLv2"
else:
# This is really only necessary for pre-1.0 versions
# of openssl, but python 2.6 doesn't expose version
# information.
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3
self.stream = SSLIOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=max_buffer_size)
else:
self.stream = IOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
max_buffer_size=max_buffer_size)
timeout = min(request.connect_timeout, request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.stream.set_close_callback(self._on_close)
self.stream.connect(
sockaddr,
functools.partial(self._on_connect, parsed, parsed_hostname))
def decode_argument(self, value, name=None):
try:
return _unicode(value)
except UnicodeDecodeError:
raise HTTPError(
400, "Invalid unicode in %s: %r" % (name or "url", value[:40]))
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size, tcp_client, max_header_size):
self.start_time = io_loop.time() # 连接开始时间
self.io_loop = io_loop
self.client = client # 请求的client，一般都是同一个
self.request = request # 代理的request
self.release_callback = release_callback # 如果取消了fetch，执行的回调
self.final_callback = final_callback # 如果结束了，执行的回调
self.max_buffer_size = max_buffer_size
self.tcp_client = tcp_client # 一个tcp的连接
self.max_header_size = max_header_size
self.code = None
self.headers = None
self.chunks = []
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
self._sockaddr = None
with stack_context.ExceptionStackContext(self._handle_exception):
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if request.allow_ipv6 is False:
af = socket.AF_INET
else:
af = socket.AF_UNSPEC
ssl_options = self._get_ssl_options(self.parsed.scheme)
timeout = min(self.request.connect_timeout,
self.request.request_timeout)
# 这里执行添加一个timeout到io_loop，也就是如果请求过时什么的
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
# 这个函数connect好像是没有回调的一个函数，最后一个callback是怎么回事儿
# 这个connect是一个被@gen.coroutine的方法
self.tcp_client.connect(host,
port,
af=af,
ssl_options=ssl_options,
callback=self._on_connect)
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size):
self.start_time = time.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.StackContext(self.cleanup):
parsed = urlparse.urlsplit(_unicode(self.request.url))
if ssl is None and parsed.scheme == "https":
raise ValueError("HTTPS requires either python2.6+ or "
"curl_httpclient")
if parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
parsed_hostname = host # save final parsed host for _on_connect
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
if request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM,
0, 0)
af, socktype, proto, canonname, sockaddr = addrinfo[0]
if parsed.scheme == "https":
ssl_options = {}
if request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if request.ca_certs is not None:
ssl_options["ca_certs"] = request.ca_certs
else:
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
if request.client_key is not None:
ssl_options["keyfile"] = request.client_key
if request.client_cert is not None:
ssl_options["certfile"] = request.client_cert
# SSL interoperability is tricky. We want to disable
# SSLv2 for security reasons; it wasn't disabled by default
# until openssl 1.0. The best way to do this is to use
# the SSL_OP_NO_SSLv2, but that wasn't exposed to python
# until 3.2. Python 2.7 adds the ciphers argument, which
# can also be used to disable SSLv2. As a last resort
# on python 2.6, we set ssl_version to SSLv3. This is
# more narrow than we'd like since it also breaks
# compatibility with servers configured for TLSv1 only,
# but nearly all servers support SSLv3:
# http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
if sys.version_info >= (2, 7):
ssl_options["ciphers"] = "DEFAULT:!SSLv2"
else:
# This is really only necessary for pre-1.0 versions
# of openssl, but python 2.6 doesn't expose version
# information.
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3
self.stream = SSLIOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=max_buffer_size)
else:
self.stream = IOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
max_buffer_size=max_buffer_size)
timeout = min(request.connect_timeout, request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
self._on_timeout)
self.stream.set_close_callback(self._on_close)
self.stream.connect(sockaddr,
functools.partial(self._on_connect, parsed,
parsed_hostname))
class _LogFormatter(logging.Formatter):
def __init__(self, color, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._color = color
if color:
# The curses module has some str/bytes confusion in
# python3. Until version 3.2.3, most methods return
# bytes, but only accept strings. In addition, we want to
# output these strings with the logging module, which
# works with unicode strings. The explicit calls to
# unicode() below are harmless in python2 but will do the
# right conversion in python 3.
fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf")
or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = unicode(fg_color, "ascii")
self._colors = {
logging.DEBUG:
unicode(
curses.tparm(fg_color, 4), # Blue
"ascii"),
logging.INFO:
unicode(
curses.tparm(fg_color, 2), # Green
"ascii"),
logging.WARNING:
unicode(
curses.tparm(fg_color, 3), # Yellow
"ascii"),
logging.ERROR:
unicode(
curses.tparm(fg_color, 1), # Red
"ascii"),
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def format(self, record):
try:
record.message = record.getMessage()
except Exception, e:
record.message = "Bad message (%r): %r" % (e, record.__dict__)
assert isinstance(record.message, basestring) # guaranteed by logging
record.asctime = datetime.datetime.fromtimestamp(
record.created).strftime('%y%m%d %H:%M:%S.%f')
prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \
record.__dict__
if self._color:
prefix = (self._colors.get(record.levelno, self._normal) + prefix +
self._normal)
# Encoding notes: The logging module prefers to work with character
# strings, but only enforces that log messages are instances of
# basestring. In python 2, non-ascii bytestrings will make
# their way through the logging framework until they blow up with
# an unhelpful decoding error (with this formatter it happens
# when we attach the prefix, but there are other opportunities for
# exceptions further along in the framework).
#
# If a byte string makes it this far, convert it to unicode to
# ensure it will make it out to the logs. Use repr() as a fallback
# to ensure that all byte strings can be converted successfully,
# but don't do it by default so we don't add extra quotes to ascii
# bytestrings. This is a bit of a hacky place to do this, but
# it's worth it since the encoding errors that would otherwise
# result are so useless (and tornado is fond of using utf8-encoded
# byte strings whereever possible).
try:
message = _unicode(record.message)
except UnicodeDecodeError:
message = repr(record.message)
formatted = prefix + " " + message
if record.exc_info:
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
formatted = formatted.rstrip() + "\n" + record.exc_text
return formatted.replace("\n", "\n ")
def html(self, template_name: str, **kwargs: Any) -> "Future[None]":
if self._finished:
raise RuntimeError("Cannot render() after finish()")
html = self.render_string(template_name, **kwargs)
# Insert the additional JS and CSS added by the modules on the page
js_embed = []
js_files = []
css_embed = []
css_files = []
html_heads = []
html_bodies = []
for module in getattr(self, "_active_modules", {}).values():
embed_part = module.embedded_javascript()
if embed_part:
js_embed.append(utf8(embed_part))
file_part = module.javascript_files()
if file_part:
if isinstance(file_part, (unicode_type, bytes)):
js_files.append(_unicode(file_part))
else:
js_files.extend(file_part)
embed_part = module.embedded_css()
if embed_part:
css_embed.append(utf8(embed_part))
file_part = module.css_files()
if file_part:
if isinstance(file_part, (unicode_type, bytes)):
css_files.append(_unicode(file_part))
else:
css_files.extend(file_part)
head_part = module.html_head()
if head_part:
html_heads.append(utf8(head_part))
body_part = module.html_body()
if body_part:
html_bodies.append(utf8(body_part))
if js_files:
# Maintain order of JavaScript files given by modules
js = self.render_linked_js(js_files)
sloc = html.rindex(b"</body>")
html = html[:sloc] + utf8(js) + b"\n" + html[sloc:]
if js_embed:
js_bytes = self.render_embed_js(js_embed)
sloc = html.rindex(b"</body>")
html = html[:sloc] + js_bytes + b"\n" + html[sloc:]
if css_files:
css = self.render_linked_css(css_files)
hloc = html.index(b"</head>")
html = html[:hloc] + utf8(css) + b"\n" + html[hloc:]
if css_embed:
css_bytes = self.render_embed_css(css_embed)
hloc = html.index(b"</head>")
html = html[:hloc] + css_bytes + b"\n" + html[hloc:]
if html_heads:
hloc = html.index(b"</head>")
html = html[:hloc] + b"".join(html_heads) + b"\n" + html[hloc:]
if html_bodies:
hloc = html.index(b"</body>")
html = html[:hloc] + b"".join(html_bodies) + b"\n" + html[hloc:]
# 就只是这里把return self.finish(html)改成了return self.write(html)
return self.write(html)
async def CAPTCHA_check(self, CAPTCHA, request_params):
try:
client = tornado.httpclient.AsyncHTTPClient()
request = {k: v for k, v in request_params.items() if (v != None)}
request = copy.deepcopy(request)
if "url_params" in request:
request["url_params"] = {
k: v.replace("__front_end_response__", CAPTCHA)
for k, v in request["url_params"].items()
}
request["url"] = request["url"] + "?" + urlencode(
request["url_params"])
del request["url_params"]
if "body_params" in request:
request["body_params"] = {
k: v.replace("__front_end_response__", CAPTCHA)
for k, v in request["body_params"].items()
}
request["body"] = urlencode(request["body_params"])
del request["body_params"]
url = request["url"]
del request["url"]
if self.block_SSRF == True:
parsed = urllib.parse.urlsplit(_unicode(url))
if parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" % url)
netloc = parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
host, port = httputil.split_host_and_port(netloc)
if port is None:
port = 443 if parsed.scheme == "https" else 80
if re.match(r"^\[.*\]$", host):
host = host[1:-1]
host_ips = list(
filter(lambda x: x[1] == socket.SOCK_STREAM,
socket.getaddrinfo(host, port)))
host_ips_global = list(
filter(lambda x: ipaddress.ip_address(x[-1][0]).is_global,
host_ips))
host_ips_connectable = [
x for x in host_ips_global if check_sock(x)
]
if len(host_ips_global) == 0:
raise self.generateError(
400, "SSRF blocked",
"Request to local network are blocked due to SSRF protection enabled"
)
if port not in self.block_SSRF_port_whitelist:
raise self.generateError(
400, "SSRF blocked",
"Request port are not in block_SSRF_port_whitelist.")
if len(host_ips_connectable) == 0:
raise self.generateError(500, "CAPTCHA API fetch error",
"Not connectable")
if "follow_redirects" in request and request[
"follow_redirects"] == True:
raise self.generateError(
400, "SSRF blocked",
"follow_redirects are not available if SSRF protection enabled"
)
request["follow_redirects"] = False
host_ip = host_ips_connectable[0]
client = tornado.httpclient.AsyncHTTPClient(
hostname_mapping={host: host_ip})
response = await client.fetch(url, **request)
self._CAPTCHA_api_response_example = response
return response
except Exception as e:
raise self.generateError(
400,
"CAPTCHA API fetch error",
e,
error_url=
"https://www.tornadoweb.org/en/stable/httpclient.html#tornado.httpclient.AsyncHTTPClient.fetch"
)
def _curl_setup_request(curl, request, buffer, headers):
curl.setopt(pycurl.URL, str(request.url)) # xxx: cannot send in unicode strings?
# libcurl's magic "Expect: 100-continue" behavior causes delays
# with servers that don't support it (which include, among others,
# Google's OpenID endpoint). Additionally, this behavior has
# a bug in conjunction with the curl_multi_socket_action API
# (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
# which increases the delays. It's more trouble than it's worth,
# so just turn off the feature (yes, setting Expect: to an empty
# value is the official way to disable this)
if "Expect" not in request.headers:
request.headers["Expect"] = ""
# libcurl adds Pragma: no-cache by default; disable that too
if "Pragma" not in request.headers:
request.headers["Pragma"] = ""
# Request headers may be either a regular dict or HTTPHeaders object
if isinstance(request.headers, httputil.HTTPHeaders):
curl.setopt(pycurl.HTTPHEADER,
[utf8("%s: %s" % i) for i in request.headers.get_all()])
else:
curl.setopt(pycurl.HTTPHEADER,
[utf8("%s: %s" % i) for i in request.headers.iteritems()])
if request.header_callback:
curl.setopt(pycurl.HEADERFUNCTION, request.header_callback)
else:
curl.setopt(pycurl.HEADERFUNCTION,
lambda line: _curl_header_callback(headers, line))
if request.streaming_callback:
curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback)
else:
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
if urlparse.urlsplit(_unicode(request.url)).scheme == 'https' and not request.validate_cert:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout))
curl.setopt(pycurl.TIMEOUT, int(request.request_timeout))
if request.user_agent:
curl.setopt(pycurl.USERAGENT, utf8(request.user_agent))
else:
curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
if request.network_interface:
curl.setopt(pycurl.INTERFACE, request.network_interface)
if request.use_gzip:
curl.setopt(pycurl.ENCODING, "gzip,deflate")
else:
curl.setopt(pycurl.ENCODING, "none")
if request.proxy_host and request.proxy_port:
curl.setopt(pycurl.PROXY, request.proxy_host)
curl.setopt(pycurl.PROXYPORT, request.proxy_port)
if request.proxy_username:
credentials = '%s:%s' % (request.proxy_username,
request.proxy_password)
curl.setopt(pycurl.PROXYUSERPWD, credentials)
else:
curl.setopt(pycurl.PROXY, '')
if request.validate_cert:
curl.setopt(pycurl.SSL_VERIFYPEER, 1)
curl.setopt(pycurl.SSL_VERIFYHOST, 2)
else:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
if request.ca_certs is not None:
curl.setopt(pycurl.CAINFO, request.ca_certs)
else:
# There is no way to restore pycurl.CAINFO to its default value
# (Using unsetopt makes it reject all certificates).
# I don't see any way to read the default value from python so it
# can be restored later. We'll have to just leave CAINFO untouched
# if no ca_certs file was specified, and require that if any
# request uses a custom ca_certs file, they all must.
pass
if request.allow_ipv6 is False:
# Curl behaves reasonably when DNS resolution gives an ipv6 address
# that we can't reach, so allow ipv6 unless the user asks to disable.
# (but see version check in _process_queue above)
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
# Set the request method through curl's retarded interface which makes
# up names for almost every single method
curl_options = {
"GET": pycurl.HTTPGET,
"POST": pycurl.POST,
"PUT": pycurl.UPLOAD,
"HEAD": pycurl.NOBODY,
}
custom_methods = set(["DELETE"])
for o in curl_options.values():
curl.setopt(o, False)
if request.method in curl_options:
curl.unsetopt(pycurl.CUSTOMREQUEST)
curl.setopt(curl_options[request.method], True)
elif request.allow_nonstandard_methods or request.method in custom_methods:
curl.setopt(pycurl.CUSTOMREQUEST, request.method)
else:
raise KeyError('unknown method ' + request.method)
# Handle curl's cryptic options for every individual HTTP method
if request.method in ("POST", "PUT"):
request_buffer = cStringIO.StringIO(utf8(request.body))
curl.setopt(pycurl.READFUNCTION, request_buffer.read)
if request.method == "POST":
def ioctl(cmd):
if cmd == curl.IOCMD_RESTARTREAD:
request_buffer.seek(0)
curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
else:
curl.setopt(pycurl.INFILESIZE, len(request.body))
logmethod = 'info' if request.log_request else 'debug'
if request.auth_username and request.auth_password:
userpwd = "%s:%s" % (request.auth_username, request.auth_password)
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
curl.setopt(pycurl.USERPWD, userpwd)
getattr(logging,logmethod)("%s %s (username: %r)", request.method, request.url,
request.auth_username)
else:
curl.unsetopt(pycurl.USERPWD)
getattr(logging,logmethod)("%s %s", request.method, request.url)
if request.client_key is not None or request.client_cert is not None:
raise ValueError("Client certificate not supported with curl_httpclient")
if threading.activeCount() > 1:
# libcurl/pycurl is not thread-safe by default. When multiple threads
# are used, signals should be disabled. This has the side effect
# of disabling DNS timeouts in some environments (when libcurl is
# not linked against ares), so we don't do it when there is only one
# thread. Applications that use many short-lived threads may need
# to set NOSIGNAL manually in a prepare_curl_callback since
# there may not be any other threads running at the time we call
# threading.activeCount.
curl.setopt(pycurl.NOSIGNAL, 1)
if request.prepare_curl_callback is not None:
request.prepare_curl_callback(curl)
def decode_argument(self, value, name=None):
try:
return _unicode(value)
except UnicodeDecodeError:
raise WSException(400, 'invalid unicode argument')
def _parse_string(self, value):
return _unicode(value)
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size):
self.start_time = time.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.code = None
self.headers = None
self.chunks = None
self._decompressor = None
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.StackContext(self.cleanup):
parsed = urlparse.urlsplit(_unicode(self.request.url))
if ssl is None and parsed.scheme == "https":
raise ValueError("HTTPS requires either python2.6+ or "
"curl_httpclient")
if parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
if match:
host = match.group(1)
port = int(match.group(2))
else:
host = netloc
port = 443 if parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
if request.allow_ipv6:
af = socket.AF_UNSPEC
else:
# We only try the first IP we get from getaddrinfo,
# so restrict to ipv4 by default.
af = socket.AF_INET
addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM,
0, 0)
af, socktype, proto, canonname, sockaddr = addrinfo[0]
if parsed.scheme == "https":
ssl_options = {}
if request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if request.ca_certs is not None:
ssl_options["ca_certs"] = request.ca_certs
else:
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
if request.client_key is not None:
ssl_options["keyfile"] = request.client_key
if request.client_cert is not None:
ssl_options["certfile"] = request.client_cert
self.stream = SSLIOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=max_buffer_size)
else:
self.stream = IOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
max_buffer_size=max_buffer_size)
timeout = min(request.connect_timeout, request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
self._on_timeout)
self.stream.set_close_callback(self._on_close)
self.stream.connect(sockaddr,
functools.partial(self._on_connect, parsed))
def run(self):
try:
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
host, port = httputil.split_host_and_port(netloc)
if port is None:
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
self.parsed_hostname = host # save final host for _on_connect
if self.request.allow_ipv6 is False:
af = socket.AF_INET
else:
af = socket.AF_UNSPEC
ssl_options = self._get_ssl_options(self.parsed.scheme)
timeout = min(self.request.connect_timeout,
self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(
functools.partial(self._on_timeout,
"while connecting")))
stream = yield self.tcp_client.connect(
host,
port,
af=af,
ssl_options=ssl_options,
max_buffer_size=self.max_buffer_size)
if self.final_callback is None:
# final_callback is cleared if we've hit our timeout.
stream.close()
return
self.stream = stream
self.stream.set_close_callback(self.on_connection_close)
self._remove_timeout()
if self.final_callback is None:
return
if self.request.request_timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + self.request.request_timeout,
stack_context.wrap(
functools.partial(self._on_timeout,
"during request")))
if (self.request.method not in self._SUPPORTED_METHODS
and not self.request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % self.request.method)
for key in ('network_interface', 'proxy_host', 'proxy_port',
'proxy_username', 'proxy_password',
'proxy_auth_mode'):
if getattr(self.request, key, None):
raise NotImplementedError('%s not supported' % key)
if "Connection" not in self.request.headers:
self.request.headers["Connection"] = "close"
if "Host" not in self.request.headers:
if '@' in self.parsed.netloc:
self.request.headers[
"Host"] = self.parsed.netloc.rpartition('@')[-1]
else:
self.request.headers["Host"] = self.parsed.netloc
username, password = None, None
if self.parsed.username is not None:
username, password = self.parsed.username, self.parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password or ''
if username is not None:
if self.request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s",
self.request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
self.request.headers["Authorization"] = (
b"Basic " + base64.b64encode(auth))
if self.request.user_agent:
self.request.headers[
"User-Agent"] = self.request.user_agent
if not self.request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
body_expected = self.request.method in ("POST", "PATCH",
"PUT")
body_present = (self.request.body is not None
or self.request.body_producer is not None)
if ((body_expected and not body_present)
or (body_present and not body_expected)):
raise ValueError(
'Body must %sbe None for method %s (unless '
'allow_nonstandard_methods is true)' %
('not ' if body_expected else '',
self.request.method))
if self.request.expect_100_continue:
self.request.headers["Expect"] = "100-continue"
if self.request.body is not None:
# When body_producer is used the caller is responsible for
# setting Content-Length (or else chunked encoding will be used).
self.request.headers["Content-Length"] = str(
len(self.request.body))
if (self.request.method == "POST"
and "Content-Type" not in self.request.headers):
self.request.headers[
"Content-Type"] = "application/x-www-form-urlencoded"
if self.request.decompress_response:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = ((self.parsed.path or '/') + (
('?' + self.parsed.query) if self.parsed.query else ''))
self.connection = self._create_connection(stream)
start_line = httputil.RequestStartLine(self.request.method,
req_path, '')
self.connection.write_headers(start_line, self.request.headers)
if self.request.expect_100_continue:
yield self.connection.read_response(self)
else:
yield self._write_body(True)
except Exception:
if not self._handle_exception(*sys.exc_info()):
raise
def prepare_request(cls, request, default_host):
parsed = urlparse.urlsplit(_unicode(request.url))
if (request.method not in cls._SUPPORTED_METHODS
and not request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % request.method)
request.follow_redirects = False
for key in (
'network_interface',
'proxy_host',
'proxy_port',
'proxy_username',
'proxy_password',
'expect_100_continue',
'body_producer',
):
if getattr(request, key, None):
raise NotImplementedError('%s not supported' % key)
request.headers.pop('Connection', None)
if "Host" not in request.headers:
if not parsed.netloc:
request.headers['Host'] = default_host
elif '@' in parsed.netloc:
request.headers["Host"] = parsed.netloc.rpartition('@')[-1]
else:
request.headers["Host"] = parsed.netloc
username, password = None, None
if parsed.username is not None:
username, password = parsed.username, parsed.password
elif request.auth_username is not None:
username = request.auth_username
password = request.auth_password or ''
if username is not None:
if request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s", request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
request.headers["Authorization"] = (b"Basic " +
base64.b64encode(auth))
if request.user_agent:
request.headers["User-Agent"] = request.user_agent
if not request.allow_nonstandard_methods:
# Some HTTP methods nearly always have bodies while others
# almost never do. Fail in this case unless the user has
# opted out of sanity checks with allow_nonstandard_methods.
body_expected = request.method in ("POST", "PATCH", "PUT")
body_present = (request.body is not None
or request.body_producer is not None)
if ((body_expected and not body_present)
or (body_present and not body_expected)):
raise ValueError(
'Body must %sbe None for method %s (unless '
'allow_nonstandard_methods is true)' %
('not ' if body_expected else '', request.method))
if request.body is not None:
# When body_producer is used the caller is responsible for
# setting Content-Length (or else chunked encoding will be used).
request.headers["Content-Length"] = str(len(request.body))
if (request.method == "POST"
and "Content-Type" not in request.headers):
request.headers[
"Content-Type"] = "application/x-www-form-urlencoded"
if request.decompress_response:
request.headers["Accept-Encoding"] = "gzip"
request.url = ((parsed.path or '/') +
(('?' + parsed.query) if parsed.query else ''))
return request
