def __after__(self): try: v = self._values # strip / to avoid treating http://example.com and http://example.com/ # as different sites v["website"] = v["website"].strip("/") user_id = self._handler.current_user.id usecase_md = markdown.markdown(escape._unicode(v["usecase"])) if v.get("site", ""): stmt = "UPDATE site SET sitename = %s,website = %s,description = %s, "\ "usecase = %s,usecase_md = %s,source_url = %s, logo = %s, "\ "updated = UTC_TIMESTAMP() where uuid_ = %s" args = (v["sitename"], v["website"], v["desc"], \ v["usecase"], usecase_md, v["source_url"], v["logo"], \ v["site"]) self._handler.db.execute(stmt, *args) else: if self._handler.db.get("select * from site where website = %s", v["website"]): self.add_error("website", "This web site already be registered.") else: if (not self._handler.is_staff): user_projects = self._handler.db.get("select count(*) as c from site where user_id = %s", \ self._handler.current_user.id) if user_projects is not None and user_projects.c >= 3: # It will need approve if someone create more than three sites status = const.Status.PENDING else: status = const.Status.UNVERIFIED else: status = const.Status.ACTIVE slug = v["website"].lower().strip() slug = _domain_prefix_re.sub("", slug) slug = unicodedata.normalize("NFKD", escape._unicode(slug)).encode("ascii", "ignore") slug = re.sub(r"[^\w]+", " ", slug) slug = "-".join(slug.split()) if not slug: slug = "site" while True: e = self._handler.db.get("SELECT * FROM site WHERE slug = %s", slug) if not e: break slug += "-" + uuid.uuid4().hex[0:2] stmt = "INSERT INTO site (sitename,website,description,usecase,usecase_md,source_url,"\ "user_id,logo,uuid_,created,updated,updated_ss,status_,slug) "\ "VALUES (%s,%s,%s,%s,%s,%s,"\ "%s,%s,%s,UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),%s,%s)" args = (v["sitename"], v["website"], v["desc"], v["usecase"], usecase_md, v["source_url"], \ user_id, v["logo"], uuid.uuid4().hex, status, slug) self._handler.db.execute(stmt, *args) except Exception, e: logging.error(str(e)) self.add_error("sitename", "Submit project error, please try it later.")
def match_list(self, match_list): arguments = self.request.arguments if '__NODELAY__' in arguments: del arguments['__NODELAY__'] if '__BLACKLIST__' in arguments: del arguments['__BLACKLIST__'] if '__ASYNCLIST__' in arguments: del arguments['__ASYNCLIST__'] if '__ASYNC_RESULT__' in arguments: del arguments['__ASYNC_RESULT__'] if '__ENABLE_DEBUG__' in arguments: del arguments['__ENABLE_DEBUG__'] if '__CONTENT_TYPE__' in arguments: del arguments['__CONTENT_TYPE__'] if '__JSONP_CALLBACK_VARNAME__' in arguments: del arguments['__JSONP_CALLBACK_VARNAME__'] match = "|".join(match_list) self.logging.info(match) p = re.compile(r'%s'%(match,),re.I|re.M) for k in arguments.keys(): arguments_string = _unicode(" ".join(arguments[k])) arguments_string = re.sub("\r|\n","",arguments_string) if p.search(arguments_string): return True if hasattr(self.request,'body'): self.logging.info(_unicode(self.request.body)) body = re.sub("\r|\n","",_unicode(self.request.body)) if p.search(body): return True return False
def respond_post(self, cmd_name): stime = time.time() user = yield objs.User.find_one( {'login_key': self.get_argument('login', '')}) self.set_header('Content-Type', 'application/json; charset=utf-8') if not cmd_name: defer.returnValue(json.dumps(dict( ok=True, commands=api_handlers.handlers.keys()))) if cmd_name not in api_handlers.handlers: defer.returnValue(json.dumps(dict( ok=False, desc='unknown command'))) callogtuple = (id(self), cmd_name, user['name'] if user else '') print "API call %d '%s' by '%s' started." % callogtuple handler = api_handlers.handlers[cmd_name] args = dict( (utf8(k), _unicode(v[0])) for k, v in self.request.arguments.iteritems()) if 'login' in args: del args['login'] self.set_header('Cache-Control', 'no-cache') cors_origin = self.request.headers.get('Origin', None) if cors_origin: self.set_header('Access-Control-Allow-Origin', '*') print "API call %d '%s' by '%s' is CORS from '%s'" % (callogtuple + (cors_origin,)) try: result = yield handler(ApiRequest(user), **args) except BnwResponse as br: print "API call %d '%s' by '%s' failed with exception. %f.s" % (callogtuple + ((time.time() - stime),)) defer.returnValue(json.dumps(dict( ok=False, desc=str(br)), ensure_ascii=False)) print "API call %d '%s' by '%s' completed. %f.s" % (callogtuple + ((time.time() - stime),)) defer.returnValue(json.dumps(result, ensure_ascii=False))
def prepare_request(cls, request, default_host): parsed = urlparse.urlsplit(_unicode(request.url)) if request.method not in cls._SUPPORTED_METHODS and not request.allow_nonstandard_methods: raise KeyError("unknown method %s" % request.method) request.follow_redirects = False for key in ( "network_interface", "proxy_host", "proxy_port", "proxy_username", "proxy_password", "expect_100_continue", "body_producer", ): if getattr(request, key, None): raise NotImplementedError("%s not supported" % key) request.headers.pop("Connection", None) if "Host" not in request.headers: if not parsed.netloc: request.headers["Host"] = default_host elif "@" in parsed.netloc: request.headers["Host"] = parsed.netloc.rpartition("@")[-1] else: request.headers["Host"] = parsed.netloc username, password = None, None if parsed.username is not None: username, password = parsed.username, parsed.password elif request.auth_username is not None: username = request.auth_username password = request.auth_password or "" if username is not None: if request.auth_mode not in (None, "basic"): raise ValueError("unsupported auth_mode %s", request.auth_mode) auth = utf8(username) + b":" + utf8(password) request.headers["Authorization"] = b"Basic " + base64.b64encode(auth) if request.user_agent: request.headers["User-Agent"] = request.user_agent if not request.allow_nonstandard_methods: # Some HTTP methods nearly always have bodies while others # almost never do. Fail in this case unless the user has # opted out of sanity checks with allow_nonstandard_methods. body_expected = request.method in ("POST", "PATCH", "PUT") body_present = request.body is not None or request.body_producer is not None if (body_expected and not body_present) or (body_present and not body_expected): raise ValueError( "Body must %sbe None for method %s (unless " "allow_nonstandard_methods is true)" % ("not " if body_expected else "", request.method) ) if request.body is not None: # When body_producer is used the caller is responsible for # setting Content-Length (or else chunked encoding will be used). request.headers["Content-Length"] = str(len(request.body)) if request.method == "POST" and "Content-Type" not in request.headers: request.headers["Content-Type"] = "application/x-www-form-urlencoded" if request.decompress_response: request.headers["Accept-Encoding"] = "gzip" request.url = (parsed.path or "/") + (("?" + parsed.query) if parsed.query else "") return request
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size, tcp_client, max_header_size): self.start_time = io_loop.time() # 连接开始时间 self.io_loop = io_loop self.client = client # 请求的client,一般都是同一个 self.request = request # 代理的request self.release_callback = release_callback # 如果取消了fetch,执行的回调 self.final_callback = final_callback # 如果结束了,执行的回调 self.max_buffer_size = max_buffer_size self.tcp_client = tcp_client # 一个tcp的连接 self.max_header_size = max_header_size self.code = None self.headers = None self.chunks = [] self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None self._sockaddr = None with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if request.allow_ipv6 is False: af = socket.AF_INET else: af = socket.AF_UNSPEC ssl_options = self._get_ssl_options(self.parsed.scheme) timeout = min(self.request.connect_timeout, self.request.request_timeout) # 这里执行添加一个timeout到io_loop,也就是如果请求过时什么的 if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap(self._on_timeout)) # 这个函数connect好像是没有回调的一个函数,最后一个callback是怎么回事儿 # 这个connect是一个被@gen.coroutine的方法 self.tcp_client.connect(host, port, af=af, ssl_options=ssl_options, callback=self._on_connect)
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size, tcp_client, max_header_size, max_body_size): self.start_time = io_loop.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.max_buffer_size = max_buffer_size self.tcp_client = tcp_client self.max_header_size = max_header_size self.max_body_size = max_body_size self.code = None self.headers = None self.chunks = [] self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None self._sockaddr = None with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) af = socket.AF_UNSPEC netloc = self.parsed.netloc host, port = httputil.split_host_and_port(netloc) timeout = min(self.request.connect_timeout, self.request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap(self._on_timeout)) self.tcp_client.connect(host,port, af=af, max_buffer_size=self.max_buffer_size, callback=self._on_connect)
def __init__(self, io_loop, client, request, callback): self.start_time = time.time() self.io_loop = io_loop self.client = client self.request = request self.callback = callback self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.StackContext(self.cleanup): parsed = urlparse.urlsplit(_unicode(self.request.url)) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r"^(.+):(\d+)$", netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if parsed.scheme == "https" else 80 if re.match(r"^\[.*\]$", host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, 0, 0) af, socktype, proto, canonname, sockaddr = addrinfo[0] if parsed.scheme == "https": ssl_options = {} if request.validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED if request.ca_certs is not None: ssl_options["ca_certs"] = request.ca_certs else: ssl_options["ca_certs"] = _DEFAULT_CA_CERTS self.stream = SSLIOStream( socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options ) else: self.stream = IOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop) timeout = min(request.connect_timeout, request.request_timeout) if timeout: self._connect_timeout = self.io_loop.add_timeout(self.start_time + timeout, self._on_timeout) self.stream.set_close_callback(self._on_close) self.stream.connect(sockaddr, functools.partial(self._on_connect, parsed))
def __call__(self, *fields, **kwargs): _params = [] _cols = [] if len(fields)<1 and not kwargs: raise database.OperationalError("Update need some date") # raise database.OperationalError, 'Update Need some data' elif kwargs: for k in kwargs.keys(): if type(kwargs[k]).__name__ == 'unicode': cond = eval(''.join(['self._queryer.', k, "==u'''", kwargs[k].replace("'''", "\\'\\'\\'"), "'''"])) elif type(kwargs[k]).__name__ == 'int' or type(kwargs[k]).__name__ == 'long': cond = eval(''.join(['self._queryer.', k, "==u'''", str(kwargs[k]), "'''"])) elif type(kwargs[k]).__name__ == 'datetime': cond = eval(''.join(['self._queryer.', k, "==u'", unicode(kwargs[k]), "'"])) else: cond = eval(''.join(['self._queryer.', k, "==u'", _unicode(kwargs[k]), "'"])) _cols.append(cond.get_sql()) _params.append(cond.get_params()[0]) else: for f in fields: _cols.append(f.get_sql()) _params.append(f.get_params()[0]) _sql_slice = ['UPDATE `', self._table, '` SET ', ','.join(_cols)] if self._where: _sql_slice.append(' WHERE ') _sql_slice.append(self._where.get_sql()) for p in self._where.get_params(): _params.append(p) _sql = ''.join(_sql_slice) return self._db.execute(_sql,*_params) raise database.OperationalError("Update need where")
def decode_argument(value, name=None): """Decodes an argument from the request. """ try: return _unicode(value) except UnicodeDecodeError: raise HTTPError(400, "Invalid unicode in %s: %r" % (name or "url", value[:40]))
def make_content(text, extra_params='rel="nofollow"'): """https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238 """ if extra_params: extra_params = " " + extra_params.strip() def make_link(m): url = m.group(1) proto = m.group(2) href = m.group(1) if not proto: href = "http://" + href params = extra_params if '.' in href: name_extension = href.split('.')[-1].lower() if name_extension in ('jpg', 'png', 'gif', 'jpeg'): return u'<img src="%s" />' % href return u'<a href="%s"%s>%s</a>' % (href, params, url) def cover_email(m): data = {'mail': m.group(1), 'end': m.group(2)} return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data def convert_mention(m): data = {} data['begin'], data['user'] = m.group(1).split('@') t = u'%(begin)s<a href="/member/%(user)s" class="mention">' \ u'@%(user)s</a>' return t % data def convert_floor(m): data = {} data['begin'], data['floor'] = m.group(1).split('#') t = u'%(begin)s<a href="#reply%(floor)s"' \ ' class="mention mention_floor">#%(floor)s</a>' return t % data def convert_topic(m): data = {} data['begin'], data['topic_link'] = m.group(1).split('t') data['topic_link_short'] = data['topic_link'][:6] t = u"""%(begin)s<a href="%(topic_link)s" class="mention mention_topic" _id=%(topic_link)s>t%(topic_link_short)s</a>""" return t % data text = _unicode(text).replace('\n', '<br />') text = text.replace(''', '\'') text = _EMAIL_RE.sub(cover_email, text) text = _MENTION_RE.sub(convert_mention, text) text = _FLOOR_RE.sub(convert_floor, text) text = _TOPIC_RE.sub(convert_topic, text) return _URL_RE.sub(make_link, text)
def post(self): self.finish( { "header": self.request.headers["X-Header-Encoding-Test"], "argument": self.get_argument("argument"), "filename": self.request.files["files"][0].filename, "filebody": _unicode(self.request.files["files"][0]["body"]), } )
def __init__(self, **kwargs): if 'form' == self.__datatype__: self._raw_data = {} # Processing on handler.request.arguments, utf-8 values for k in kwargs: if isinstance(kwargs[k], list): if len(kwargs[k]) > 1: self._raw_data[k] = map(_unicode, kwargs[k]) else: self._raw_data[k] = _unicode(kwargs[k][0]) else: self._raw_data[k] = _unicode(kwargs[k]) else: self._raw_data = kwargs self.data = {} self.errors = [] self.validate()
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size, resolver): self.start_time = io_loop.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.max_buffer_size = max_buffer_size self.resolver = resolver self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET # 支持request的timeout timeout = min(self.request.connect_timeout, self.request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap(self._on_timeout)) # dns解析模块,解析ip地址 # Tornado 本身提供两个dns解析库,一个是基于多线程的,一个就是直接同步的blocking版本 self.resolver.resolve(host, port, af, callback=self._on_resolve)
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size, tcp_client, max_header_size, max_body_size): self.start_time = io_loop.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.max_buffer_size = max_buffer_size self.tcp_client = tcp_client self.max_header_size = max_header_size self.max_body_size = max_body_size self.code = None self.headers = None self.chunks = [] self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None self._sockaddr = None with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") host, port = httputil.split_host_and_port(netloc) if port is None: port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if request.allow_ipv6 is False: af = socket.AF_INET else: af = socket.AF_UNSPEC ssl_options = self._get_ssl_options(self.parsed.scheme) timeout = min(self.request.connect_timeout, self.request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap(self._on_timeout)) self.tcp_client.connect(host, port, af=af, ssl_options=ssl_options, max_buffer_size=self.max_buffer_size, callback=self._on_connect)
def getlist(self, name, strip=True): values = [] for v in self.get(name, []): v = _unicode(v) if isinstance(v, unicode): # Get rid of any weird control chars (unless decoding gave # us bytes, in which case leave it alone) v = re.sub(r"[\x00-\x08\x0e-\x1f]", " ", v) if strip: v = v.strip() values.append(v) return values
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size): self.start_time = io_loop.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.max_buffer_size = max_buffer_size self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if ssl is None and self.parsed.scheme == "https": raise ValueError("HTTPS requires either python2.6+ or " "curl_httpclient") if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET self.client.resolver.getaddrinfo( host, port, af, socket.SOCK_STREAM, 0, 0, callback=self._on_resolve)
def __init__(self, **kwargs): self._raw_data = {} # handler.request.arguments, utf-8 values for k in kwargs: if isinstance(kwargs[k], list): self._raw_data[k] = map(_unicode, kwargs[k]) else: self._raw_data[k] = _unicode(kwargs[k]) self.data = {} self.errors = [] self.validate()
def make_content(text, extra_params='rel="nofollow"'): """https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238 """ if extra_params: extra_params = " " + extra_params.strip() def make_link(m): url = m.group(1) proto = m.group(2) href = m.group(1) if not proto: href = "http://" + href # no proto specified, use http params = extra_params if '.' in href: name_extension = href.split('.')[-1].lower() if name_extension in ('jpg', 'png', 'git', 'jpeg'): return u'<img src="%s" />' % href return u'<a href="%s"%s>%s</a>' % (href, params, url) def cover_email(m): data = {'mail': m.group(1), 'end': m.group(2)} return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data def convert_mention(m): data = {} data['begin'], data['user'] = m.group(1).split('@') t = u'%(begin)s<a href="/member/%(user)s" class="mention">' \ u'@%(user)s</a>' return t % data def highligt(m): try: name = m.group(1) lexer = get_lexer_by_name(name) except ValueError: lexer = TextLexer() text = m.group(2).replace('"', '"').replace('&', '&') text = text.replace('<', '<').replace('>', '>') text = text.replace(' ', ' ') return highlight(text, lexer, formatter) text = _unicode(xhtml_escape(text)).replace(' ', ' ') text = _CODE_RE.sub(highligt, text).replace('\n', '<br />') text = _EMAIL_RE.sub(cover_email, text) text = _MENTION_RE.sub(convert_mention, text) return _URL_RE.sub(make_link, text)
def match_list(self, match_list): arguments = self.request.arguments if '__NODELAY__' in arguments: del arguments['__NODELAY__'] if '__BLACKLIST__' in arguments: del arguments['__BLACKLIST__'] if '__ASYNCLIST__' in arguments: del arguments['__ASYNCLIST__'] if '__ASYNC_RESULT__' in arguments: del arguments['__ASYNC_RESULT__'] if '__ENABLE_DEBUG__' in arguments: del arguments['__ENABLE_DEBUG__'] if '__CONTENT_TYPE__' in arguments: del arguments['__CONTENT_TYPE__'] if '__JSONP_CALLBACK_VARNAME__' in arguments: del arguments['__JSONP_CALLBACK_VARNAME__'] match = "|".join(match_list) for k in arguments.keys(): if re.match(match,_unicode(" ".join(arguments[k]))): return True if server_request.body != '' and re.match(match,_unicode(server_request.body)): return True return False
def make_content(text, extra_params='rel="nofollow"'): """https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238 """ if extra_params: extra_params = " " + extra_params.strip() def make_link(m): url = m.group(1) proto = m.group(2) href = m.group(1) if not proto: href = "http://" + href # no proto specified, use http params = extra_params if "." in href: name_extension = href.split(".")[-1] if name_extension in ("jpg", "png", "git", "jpeg"): return u'<img src="%s" />' % href return u'<a href="%s"%s>%s</a>' % (href, params, url) def cover_email(m): data = {"mail": m.group(1), "end": m.group(2)} return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data def convert_mention(m): data = {} data["begin"], data["user"] = m.group(1).split("@") t = u'%(begin)s<a href="/member/%(user)s" class="mention">' u"@%(user)s</a>" return t % data def highligt(m): try: name = m.group(1) lexer = get_lexer_by_name(name) except ValueError: lexer = TextLexer() text = m.group(2).replace(""", '"').replace("&", "&") text = text.replace("<", "<").replace(">", ">") return highlight(text, lexer, formatter) text = _unicode(xhtml_escape(text)) text = _CODE_RE.sub(highligt, text).replace("\n", "<br />") text = _EMAIL_RE.sub(cover_email, text) text = _MENTION_RE.sub(convert_mention, text) return _URL_RE.sub(make_link, text)
def resolve(self): self.proxy_host = getattr(self.request, 'proxy_host', None) self.proxy_port = getattr(self.request, 'proxy_port', 8080) if self.proxy_host: host = self.proxy_host port = self.proxy_port self.is_support_keepalive = False with stack_context.ExceptionStackContext(self._handle_exception): self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if self.request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET self.update_timeout() if self.is_connected(): self.logger.info('is connecting ....') self._on_connect() else: if not self.proxy_host: self.resolver.resolve(host, port, af, callback=self._on_resolve) else: self.resolver.resolve(self.proxy_host, self.proxy_port, af, callback=self._on_resolve)
def test_url_fragment(self): # Even though [RFC 3986](http://tools.ietf.org/html/rfc3986#section-3.5) specifically state that: # "...the fragment identifier is not used in the scheme-specific processing of a URI; instead, # the fragment identifier is separated from the rest of the URI prior to a dereference, and thus # the identifying information within the fragment itself is dereferenced solely by the user agent, # regardless of the URI scheme." # It is somewhat dubious on whether a web framework like tornado should ignore the fragment. # Users might need it to decide on what to serve. Consider this other part of the spec: # "As with any URI, use of a fragment identifier component does not imply that a retrieval action # will take place. A URI with a fragment identifier may be used to refer to the secondary resource # without any implication that the primary resource is accessible or will ever be accessed." # All of this considered, this change only allows users to use fragments if they do see fit. response = self.fetch("/echofrag#frag=123") self.assertEqual(200, response.code) frag, sep, query = _unicode(response.body).partition('\n') self.assertEqual("frag=123", frag)
def getlist(self, key): """ tornado.web.RequestHandler._get_arguments """ try: _remove_control_chars_regex = re.compile(r"[\x00-\x08\x0e-\x1f]") values = [] for v in self.get(key, []): if isinstance(v, bytes): v = _unicode(v) if isinstance(v, unicode_type): v = _remove_control_chars_regex.sub(" ", v) values.append(v) return values except UnicodeDecodeError: return []
def __init__(self, io_loop, client, request, callback): self.start_time = time.time() self.io_loop = io_loop self.client = client self.request = request self.callback = callback self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.StackContext(self.cleanup): parsed = urlparse.urlsplit(_unicode(self.request.url)) host = parsed.hostname if parsed.port is None: port = 443 if parsed.scheme == "https" else 80 else: port = parsed.port if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if parsed.scheme == "https": ssl_options = {} if request.validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED if request.ca_certs is not None: ssl_options["ca_certs"] = request.ca_certs else: ssl_options["ca_certs"] = _DEFAULT_CA_CERTS self.stream = SSLIOStream(socket.socket(), io_loop=self.io_loop, ssl_options=ssl_options) else: self.stream = IOStream(socket.socket(), io_loop=self.io_loop) timeout = min(request.connect_timeout, request.request_timeout) if timeout: self._connect_timeout = self.io_loop.add_timeout( self.start_time + timeout, self._on_timeout) self.stream.set_close_callback(self._on_close) self.stream.connect((host, port), functools.partial(self._on_connect, parsed))
def format(self, record): try: record.message = record.getMessage() except Exception as e: record.message = "Bad message (%r): %r" % (e, record.__dict__) assert isinstance(record.message, basestring_type) # guaranteed by logging record.asctime = time.strftime( "%y%m%d %H:%M:%S", self.converter(record.created)) prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ record.__dict__ if self._color: prefix = (self._colors.get(record.levelno, self._normal) + prefix + self._normal) # Encoding notes: The logging module prefers to work with character # strings, but only enforces that log messages are instances of # basestring. In python 2, non-ascii bytestrings will make # their way through the logging framework until they blow up with # an unhelpful decoding error (with this formatter it happens # when we attach the prefix, but there are other opportunities for # exceptions further along in the framework). # # If a byte string makes it this far, convert it to unicode to # ensure it will make it out to the logs. Use repr() as a fallback # to ensure that all byte strings can be converted successfully, # but don't do it by default so we don't add extra quotes to ascii # bytestrings. This is a bit of a hacky place to do this, but # it's worth it since the encoding errors that would otherwise # result are so useless (and tornado is fond of using utf8-encoded # byte strings whereever possible). try: message = _unicode(record.message) except UnicodeDecodeError: message = repr(record.message) formatted = prefix + " " + message if record.exc_info: if not record.exc_text: record.exc_text = self.formatException(record.exc_info) if record.exc_text: formatted = formatted.rstrip() + "\n" + record.exc_text return formatted.replace("\n", "\n ")
def decode_argument(self, value, name=None): """Decodes an argument from the command parameters. The argument has been percent-decoded and is now a byte string. By default, this method decodes the argument as utf-8 and returns a unicode string, but this may be overridden in subclasses. This method is used as a filter for both `get_argument()` and for values extracted from the parameters The name of the argument is provided if known, but may be None (e.g. for unnamed groups in the url regex). """ if not isinstance(value, (unicode_type,)): return value try: return _unicode(value) except UnicodeDecodeError: raise CommandError("Invalid unicode in %s: %r" % (name or "", value[:40]))
def format(self, text, secure=False): #def linkshit_format(text, secure=False, parser=moin_parser): text = _unicode(xhtml_escape(text)) out = [] thumbs = [] stack = [] instrong = False inemph = False self.formatting_tags.keys() for m in self.parser.parse(text): if isinstance(m, tuple): if m[0] in self._formatfuncs: new_html, new_thumb = self._formatfuncs[m[0]](m, secure) if new_html: out.append(new_html) if new_thumb: thumbs.append(new_thumb) elif m[0] in self.formatting_tags: tag = self.formatting_tags[m[0]] if not m[0] in stack: out.append(tag[0]) stack.insert(0, m[0]) else: tp = stack.index(m[0]) for uptag in stack[:tp + 1]: out.append(self.formatting_tags[uptag][1]) for uptag in reversed(stack[:tp]): out.append(self.formatting_tags[uptag][0]) del stack[tp] else: out.append('%s<!-- %s -->' % (m[1], m[0])) else: out.append(m) for i in stack: out.append(self.formatting_tags[i][1]) return '<div class="pwls">'+''.join(out)+'</div>', thumbs
def _parse_string(self, value): return _unicode(value)
def decode_argument(self, value): return _unicode(value)
async def run(self) -> None: try: self.parsed = urllib.parse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") host, port = httputil.split_host_and_port(netloc) if port is None: port = 443 if self.parsed.scheme == "https" else 80 if re.match(r"^\[.*\]$", host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if self.request.allow_ipv6 is False: af = socket.AF_INET else: af = socket.AF_UNSPEC ssl_options = self._get_ssl_options(self.parsed.scheme) source_ip = None if self.request.network_interface: if is_valid_ip(self.request.network_interface): source_ip = self.request.network_interface else: raise ValueError( "Unrecognized IPv4 or IPv6 address for network_interface, got %r" % (self.request.network_interface,) ) timeout = ( min(self.request.connect_timeout, self.request.request_timeout) or self.request.connect_timeout or self.request.request_timeout ) # min but skip zero if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, functools.partial(self._on_timeout, "while connecting"), ) stream = await self.tcp_client.connect( host, port, af=af, ssl_options=ssl_options, max_buffer_size=self.max_buffer_size, source_ip=source_ip, ) if self.final_callback is None: # final_callback is cleared if we've hit our timeout. stream.close() return self.stream = stream self.stream.set_close_callback(self.on_connection_close) self._remove_timeout() if self.final_callback is None: return if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, functools.partial(self._on_timeout, "during request"), ) if ( self.request.method not in self._SUPPORTED_METHODS and not self.request.allow_nonstandard_methods ): raise KeyError("unknown method %s" % self.request.method) for key in ( "proxy_host", "proxy_port", "proxy_username", "proxy_password", "proxy_auth_mode", ): if getattr(self.request, key, None): raise NotImplementedError("%s not supported" % key) if "Connection" not in self.request.headers: self.request.headers["Connection"] = "close" if "Host" not in self.request.headers: if "@" in self.parsed.netloc: self.request.headers["Host"] = self.parsed.netloc.rpartition("@")[ -1 ] else: self.request.headers["Host"] = self.parsed.netloc username, password = None, None if self.parsed.username is not None: username, password = self.parsed.username, self.parsed.password elif self.request.auth_username is not None: username = self.request.auth_username password = self.request.auth_password or "" if username is not None: assert password is not None if self.request.auth_mode not in (None, "basic"): raise ValueError("unsupported auth_mode %s", self.request.auth_mode) self.request.headers["Authorization"] = "Basic " + _unicode( base64.b64encode( httputil.encode_username_password(username, password) ) ) if self.request.user_agent: self.request.headers["User-Agent"] = self.request.user_agent elif self.request.headers.get("User-Agent") is None: self.request.headers["User-Agent"] = "Tornado/{}".format(version) if not self.request.allow_nonstandard_methods: # Some HTTP methods nearly always have bodies while others # almost never do. Fail in this case unless the user has # opted out of sanity checks with allow_nonstandard_methods. body_expected = self.request.method in ("POST", "PATCH", "PUT") body_present = ( self.request.body is not None or self.request.body_producer is not None ) if (body_expected and not body_present) or ( body_present and not body_expected ): raise ValueError( "Body must %sbe None for method %s (unless " "allow_nonstandard_methods is true)" % ("not " if body_expected else "", self.request.method) ) if self.request.expect_100_continue: self.request.headers["Expect"] = "100-continue" if self.request.body is not None: # When body_producer is used the caller is responsible for # setting Content-Length (or else chunked encoding will be used). self.request.headers["Content-Length"] = str(len(self.request.body)) if ( self.request.method == "POST" and "Content-Type" not in self.request.headers ): self.request.headers[ "Content-Type" ] = "application/x-www-form-urlencoded" if self.request.decompress_response: self.request.headers["Accept-Encoding"] = "gzip" req_path = (self.parsed.path or "/") + ( ("?" + self.parsed.query) if self.parsed.query else "" ) self.connection = self._create_connection(stream) start_line = httputil.RequestStartLine(self.request.method, req_path, "") self.connection.write_headers(start_line, self.request.headers) if self.request.expect_100_continue: await self.connection.read_response(self) else: await self._write_body(True) except Exception: if not self._handle_exception(*sys.exc_info()): raise
def safe_unicode(s): try: return _unicode(s) except UnicodeDecodeError: return repr(s)
def make_content(text, extra_params='rel="nofollow"'): """https://github.com/facebook/tornado/blob/master/tornado/escape.py#L238 """ if extra_params: extra_params = " " + extra_params.strip() def make_link(m): url = m.group(1) proto = m.group(2) href = m.group(1) if not proto: href = "http://" + href # no proto specified, use http params = extra_params if '.' in href: name_extension = href.split('.')[-1].lower() if name_extension in ('jpg', 'png', 'git', 'jpeg'): return u'<img src="%s" />' % href return u'<a href="%s"%s>%s</a>' % (href, params, url) def cover_email(m): data = {'mail': m.group(1), 'end': m.group(2)} return u'<a href="mailto:%(mail)s">%(mail)s</a>%(end)s' % data def convert_mention(m): data = {} data['begin'], data['user'] = m.group(1).split('@') t = u'%(begin)s<a href="/member/%(user)s" class="mention">' \ u'@%(user)s</a>' return t % data def convert_floor(m): data = {} data['begin'], data['floor'] = m.group(1).split('#') t = u'%(begin)s<a href="#reply%(floor)s"' \ ' class="mention mention_floor">#%(floor)s</a>' return t % data def convert_topic(m): data = {} data['begin'], data['topic_link'] = m.group(1).split('t') data['topic_link_short'] = data['topic_link'][:6] t = u"""%(begin)s<a href="%(topic_link)s" class="mention mention_topic" _id=%(topic_link)s>t%(topic_link_short)s</a>""" return t % data def highligt(m): try: name = m.group(1) lexer = get_lexer_by_name(name) except ValueError: lexer = TextLexer() text = m.group(2).replace('"', '"').replace('&', '&') text = text.replace('<', '<').replace('>', '>') text = text.replace(' ', ' ') return highlight(text, lexer, formatter) text = _unicode(xhtml_escape(text)).replace(' ', ' ') text = _CODE_RE.sub(highligt, text).replace('\n', '<br />') text = _EMAIL_RE.sub(cover_email, text) text = _MENTION_RE.sub(convert_mention, text) text = _FLOOR_RE.sub(convert_floor, text) text = _TOPIC_RE.sub(convert_topic, text) return _URL_RE.sub(make_link, text)
def __after__(self): try: v = self._values # strip / to avoid treating http://example.com and http://example.com/ # as different sites v["website"] = v["website"].strip("/") user_id = self._handler.current_user.id usecase_md = markdown.markdown(escape._unicode(v["usecase"])) if v.get("site", ""): stmt = "UPDATE site SET sitename = %s,website = %s,description = %s, "\ "usecase = %s,usecase_md = %s,source_url = %s, logo = %s, "\ "updated = UTC_TIMESTAMP() where uuid_ = %s" args = (v["sitename"], v["website"], v["desc"], \ v["usecase"], usecase_md, v["source_url"], v["logo"], \ v["site"]) self._handler.db.execute(stmt, *args) else: if self._handler.db.get( "select * from site where website = %s", v["website"]): self.add_error("website", "This web site already be registered.") else: if (not self._handler.is_staff): user_projects = self._handler.db.get("select count(*) as c from site where user_id = %s", \ self._handler.current_user.id) if user_projects is not None and user_projects.c >= 3: # It will need approve if someone create more than three sites status = const.Status.PENDING else: status = const.Status.UNVERIFIED else: status = const.Status.ACTIVE slug = v["website"].lower().strip() slug = _domain_prefix_re.sub("", slug) slug = unicodedata.normalize("NFKD", escape._unicode(slug)).encode( "ascii", "ignore") slug = re.sub(r"[^\w]+", " ", slug) slug = "-".join(slug.split()) if not slug: slug = "site" while True: e = self._handler.db.get( "SELECT * FROM site WHERE slug = %s", slug) if not e: break slug += "-" + uuid.uuid4().hex[0:2] stmt = "INSERT INTO site (sitename,website,description,usecase,usecase_md,source_url,"\ "user_id,logo,uuid_,created,updated,updated_ss,status_,slug) "\ "VALUES (%s,%s,%s,%s,%s,%s,"\ "%s,%s,%s,UTC_TIMESTAMP(),UTC_TIMESTAMP(),UTC_TIMESTAMP(),%s,%s)" args = (v["sitename"], v["website"], v["desc"], v["usecase"], usecase_md, v["source_url"], \ user_id, v["logo"], uuid.uuid4().hex, status, slug) self._handler.db.execute(stmt, *args) except Exception, e: logging.error(str(e)) self.add_error("sitename", "Submit project error, please try it later.")
def _safe_unicode(s): try: return _unicode(s) except UnicodeDecodeError: return repr(s)
def _parse_string(self, value: str) -> str: return _unicode(value)
def _safe_unicode(s: Any) -> str: try: return _unicode(s) except UnicodeDecodeError: return repr(s)
def post(self): self.finish({"header": self.request.headers["X-Header-Encoding-Test"], "argument": self.get_argument("argument"), "filename": self.request.files["files"][0].filename, "filebody": _unicode(self.request.files["files"][0]["body"]), })
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size): self.start_time = time.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.StackContext(self.cleanup): parsed = urllib.parse.urlsplit(_unicode(self.request.url)) if ssl is None and parsed.scheme == "https": raise ValueError("HTTPS requires either python2.6+ or " "curl_httpclient") if parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] parsed_hostname = host # save final parsed host for _on_connect if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, 0, 0) af, socktype, proto, canonname, sockaddr = addrinfo[0] if parsed.scheme == "https": ssl_options = {} if request.validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED if request.ca_certs is not None: ssl_options["ca_certs"] = request.ca_certs else: ssl_options["ca_certs"] = _DEFAULT_CA_CERTS if request.client_key is not None: ssl_options["keyfile"] = request.client_key if request.client_cert is not None: ssl_options["certfile"] = request.client_cert # SSL interoperability is tricky. We want to disable # SSLv2 for security reasons; it wasn't disabled by default # until openssl 1.0. The best way to do this is to use # the SSL_OP_NO_SSLv2, but that wasn't exposed to python # until 3.2. Python 2.7 adds the ciphers argument, which # can also be used to disable SSLv2. As a last resort # on python 2.6, we set ssl_version to SSLv3. This is # more narrow than we'd like since it also breaks # compatibility with servers configured for TLSv1 only, # but nearly all servers support SSLv3: # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html if sys.version_info >= (2, 7): ssl_options["ciphers"] = "DEFAULT:!SSLv2" else: # This is really only necessary for pre-1.0 versions # of openssl, but python 2.6 doesn't expose version # information. ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 self.stream = SSLIOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options, max_buffer_size=max_buffer_size) else: self.stream = IOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, max_buffer_size=max_buffer_size) timeout = min(request.connect_timeout, request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap(self._on_timeout)) self.stream.set_close_callback(self._on_close) self.stream.connect( sockaddr, functools.partial(self._on_connect, parsed, parsed_hostname))
def decode_argument(self, value, name=None): try: return _unicode(value) except UnicodeDecodeError: raise HTTPError( 400, "Invalid unicode in %s: %r" % (name or "url", value[:40]))
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size): self.start_time = time.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.StackContext(self.cleanup): parsed = urlparse.urlsplit(_unicode(self.request.url)) if ssl is None and parsed.scheme == "https": raise ValueError("HTTPS requires either python2.6+ or " "curl_httpclient") if parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] parsed_hostname = host # save final parsed host for _on_connect if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, 0, 0) af, socktype, proto, canonname, sockaddr = addrinfo[0] if parsed.scheme == "https": ssl_options = {} if request.validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED if request.ca_certs is not None: ssl_options["ca_certs"] = request.ca_certs else: ssl_options["ca_certs"] = _DEFAULT_CA_CERTS if request.client_key is not None: ssl_options["keyfile"] = request.client_key if request.client_cert is not None: ssl_options["certfile"] = request.client_cert # SSL interoperability is tricky. We want to disable # SSLv2 for security reasons; it wasn't disabled by default # until openssl 1.0. The best way to do this is to use # the SSL_OP_NO_SSLv2, but that wasn't exposed to python # until 3.2. Python 2.7 adds the ciphers argument, which # can also be used to disable SSLv2. As a last resort # on python 2.6, we set ssl_version to SSLv3. This is # more narrow than we'd like since it also breaks # compatibility with servers configured for TLSv1 only, # but nearly all servers support SSLv3: # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html if sys.version_info >= (2, 7): ssl_options["ciphers"] = "DEFAULT:!SSLv2" else: # This is really only necessary for pre-1.0 versions # of openssl, but python 2.6 doesn't expose version # information. ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 self.stream = SSLIOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options, max_buffer_size=max_buffer_size) else: self.stream = IOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, max_buffer_size=max_buffer_size) timeout = min(request.connect_timeout, request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, self._on_timeout) self.stream.set_close_callback(self._on_close) self.stream.connect(sockaddr, functools.partial(self._on_connect, parsed, parsed_hostname))
class _LogFormatter(logging.Formatter): def __init__(self, color, *args, **kwargs): logging.Formatter.__init__(self, *args, **kwargs) self._color = color if color: # The curses module has some str/bytes confusion in # python3. Until version 3.2.3, most methods return # bytes, but only accept strings. In addition, we want to # output these strings with the logging module, which # works with unicode strings. The explicit calls to # unicode() below are harmless in python2 but will do the # right conversion in python 3. fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = unicode(fg_color, "ascii") self._colors = { logging.DEBUG: unicode( curses.tparm(fg_color, 4), # Blue "ascii"), logging.INFO: unicode( curses.tparm(fg_color, 2), # Green "ascii"), logging.WARNING: unicode( curses.tparm(fg_color, 3), # Yellow "ascii"), logging.ERROR: unicode( curses.tparm(fg_color, 1), # Red "ascii"), } self._normal = unicode(curses.tigetstr("sgr0"), "ascii") def format(self, record): try: record.message = record.getMessage() except Exception, e: record.message = "Bad message (%r): %r" % (e, record.__dict__) assert isinstance(record.message, basestring) # guaranteed by logging record.asctime = datetime.datetime.fromtimestamp( record.created).strftime('%y%m%d %H:%M:%S.%f') prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ record.__dict__ if self._color: prefix = (self._colors.get(record.levelno, self._normal) + prefix + self._normal) # Encoding notes: The logging module prefers to work with character # strings, but only enforces that log messages are instances of # basestring. In python 2, non-ascii bytestrings will make # their way through the logging framework until they blow up with # an unhelpful decoding error (with this formatter it happens # when we attach the prefix, but there are other opportunities for # exceptions further along in the framework). # # If a byte string makes it this far, convert it to unicode to # ensure it will make it out to the logs. Use repr() as a fallback # to ensure that all byte strings can be converted successfully, # but don't do it by default so we don't add extra quotes to ascii # bytestrings. This is a bit of a hacky place to do this, but # it's worth it since the encoding errors that would otherwise # result are so useless (and tornado is fond of using utf8-encoded # byte strings whereever possible). try: message = _unicode(record.message) except UnicodeDecodeError: message = repr(record.message) formatted = prefix + " " + message if record.exc_info: if not record.exc_text: record.exc_text = self.formatException(record.exc_info) if record.exc_text: formatted = formatted.rstrip() + "\n" + record.exc_text return formatted.replace("\n", "\n ")
def html(self, template_name: str, **kwargs: Any) -> "Future[None]": if self._finished: raise RuntimeError("Cannot render() after finish()") html = self.render_string(template_name, **kwargs) # Insert the additional JS and CSS added by the modules on the page js_embed = [] js_files = [] css_embed = [] css_files = [] html_heads = [] html_bodies = [] for module in getattr(self, "_active_modules", {}).values(): embed_part = module.embedded_javascript() if embed_part: js_embed.append(utf8(embed_part)) file_part = module.javascript_files() if file_part: if isinstance(file_part, (unicode_type, bytes)): js_files.append(_unicode(file_part)) else: js_files.extend(file_part) embed_part = module.embedded_css() if embed_part: css_embed.append(utf8(embed_part)) file_part = module.css_files() if file_part: if isinstance(file_part, (unicode_type, bytes)): css_files.append(_unicode(file_part)) else: css_files.extend(file_part) head_part = module.html_head() if head_part: html_heads.append(utf8(head_part)) body_part = module.html_body() if body_part: html_bodies.append(utf8(body_part)) if js_files: # Maintain order of JavaScript files given by modules js = self.render_linked_js(js_files) sloc = html.rindex(b"</body>") html = html[:sloc] + utf8(js) + b"\n" + html[sloc:] if js_embed: js_bytes = self.render_embed_js(js_embed) sloc = html.rindex(b"</body>") html = html[:sloc] + js_bytes + b"\n" + html[sloc:] if css_files: css = self.render_linked_css(css_files) hloc = html.index(b"</head>") html = html[:hloc] + utf8(css) + b"\n" + html[hloc:] if css_embed: css_bytes = self.render_embed_css(css_embed) hloc = html.index(b"</head>") html = html[:hloc] + css_bytes + b"\n" + html[hloc:] if html_heads: hloc = html.index(b"</head>") html = html[:hloc] + b"".join(html_heads) + b"\n" + html[hloc:] if html_bodies: hloc = html.index(b"</body>") html = html[:hloc] + b"".join(html_bodies) + b"\n" + html[hloc:] # 就只是这里把return self.finish(html)改成了return self.write(html) return self.write(html)
async def CAPTCHA_check(self, CAPTCHA, request_params): try: client = tornado.httpclient.AsyncHTTPClient() request = {k: v for k, v in request_params.items() if (v != None)} request = copy.deepcopy(request) if "url_params" in request: request["url_params"] = { k: v.replace("__front_end_response__", CAPTCHA) for k, v in request["url_params"].items() } request["url"] = request["url"] + "?" + urlencode( request["url_params"]) del request["url_params"] if "body_params" in request: request["body_params"] = { k: v.replace("__front_end_response__", CAPTCHA) for k, v in request["body_params"].items() } request["body"] = urlencode(request["body_params"]) del request["body_params"] url = request["url"] del request["url"] if self.block_SSRF == True: parsed = urllib.parse.urlsplit(_unicode(url)) if parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % url) netloc = parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") host, port = httputil.split_host_and_port(netloc) if port is None: port = 443 if parsed.scheme == "https" else 80 if re.match(r"^\[.*\]$", host): host = host[1:-1] host_ips = list( filter(lambda x: x[1] == socket.SOCK_STREAM, socket.getaddrinfo(host, port))) host_ips_global = list( filter(lambda x: ipaddress.ip_address(x[-1][0]).is_global, host_ips)) host_ips_connectable = [ x for x in host_ips_global if check_sock(x) ] if len(host_ips_global) == 0: raise self.generateError( 400, "SSRF blocked", "Request to local network are blocked due to SSRF protection enabled" ) if port not in self.block_SSRF_port_whitelist: raise self.generateError( 400, "SSRF blocked", "Request port are not in block_SSRF_port_whitelist.") if len(host_ips_connectable) == 0: raise self.generateError(500, "CAPTCHA API fetch error", "Not connectable") if "follow_redirects" in request and request[ "follow_redirects"] == True: raise self.generateError( 400, "SSRF blocked", "follow_redirects are not available if SSRF protection enabled" ) request["follow_redirects"] = False host_ip = host_ips_connectable[0] client = tornado.httpclient.AsyncHTTPClient( hostname_mapping={host: host_ip}) response = await client.fetch(url, **request) self._CAPTCHA_api_response_example = response return response except Exception as e: raise self.generateError( 400, "CAPTCHA API fetch error", e, error_url= "https://www.tornadoweb.org/en/stable/httpclient.html#tornado.httpclient.AsyncHTTPClient.fetch" )
def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.URL, str(request.url)) # xxx: cannot send in unicode strings? # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, # Google's OpenID endpoint). Additionally, this behavior has # a bug in conjunction with the curl_multi_socket_action API # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), # which increases the delays. It's more trouble than it's worth, # so just turn off the feature (yes, setting Expect: to an empty # value is the official way to disable this) if "Expect" not in request.headers: request.headers["Expect"] = "" # libcurl adds Pragma: no-cache by default; disable that too if "Pragma" not in request.headers: request.headers["Pragma"] = "" # Request headers may be either a regular dict or HTTPHeaders object if isinstance(request.headers, httputil.HTTPHeaders): curl.setopt(pycurl.HTTPHEADER, [utf8("%s: %s" % i) for i in request.headers.get_all()]) else: curl.setopt(pycurl.HTTPHEADER, [utf8("%s: %s" % i) for i in request.headers.iteritems()]) if request.header_callback: curl.setopt(pycurl.HEADERFUNCTION, request.header_callback) else: curl.setopt(pycurl.HEADERFUNCTION, lambda line: _curl_header_callback(headers, line)) if request.streaming_callback: curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback) else: curl.setopt(pycurl.WRITEFUNCTION, buffer.write) if urlparse.urlsplit(_unicode(request.url)).scheme == 'https' and not request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout)) curl.setopt(pycurl.TIMEOUT, int(request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, utf8(request.user_agent)) else: curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: curl.setopt(pycurl.INTERFACE, request.network_interface) if request.use_gzip: curl.setopt(pycurl.ENCODING, "gzip,deflate") else: curl.setopt(pycurl.ENCODING, "none") if request.proxy_host and request.proxy_port: curl.setopt(pycurl.PROXY, request.proxy_host) curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: credentials = '%s:%s' % (request.proxy_username, request.proxy_password) curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) else: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) if request.ca_certs is not None: curl.setopt(pycurl.CAINFO, request.ca_certs) else: # There is no way to restore pycurl.CAINFO to its default value # (Using unsetopt makes it reject all certificates). # I don't see any way to read the default value from python so it # can be restored later. We'll have to just leave CAINFO untouched # if no ca_certs file was specified, and require that if any # request uses a custom ca_certs file, they all must. pass if request.allow_ipv6 is False: # Curl behaves reasonably when DNS resolution gives an ipv6 address # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) # Set the request method through curl's retarded interface which makes # up names for almost every single method curl_options = { "GET": pycurl.HTTPGET, "POST": pycurl.POST, "PUT": pycurl.UPLOAD, "HEAD": pycurl.NOBODY, } custom_methods = set(["DELETE"]) for o in curl_options.values(): curl.setopt(o, False) if request.method in curl_options: curl.unsetopt(pycurl.CUSTOMREQUEST) curl.setopt(curl_options[request.method], True) elif request.allow_nonstandard_methods or request.method in custom_methods: curl.setopt(pycurl.CUSTOMREQUEST, request.method) else: raise KeyError('unknown method ' + request.method) # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): request_buffer = cStringIO.StringIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": def ioctl(cmd): if cmd == curl.IOCMD_RESTARTREAD: request_buffer.seek(0) curl.setopt(pycurl.IOCTLFUNCTION, ioctl) curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) logmethod = 'info' if request.log_request else 'debug' if request.auth_username and request.auth_password: userpwd = "%s:%s" % (request.auth_username, request.auth_password) curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) curl.setopt(pycurl.USERPWD, userpwd) getattr(logging,logmethod)("%s %s (username: %r)", request.method, request.url, request.auth_username) else: curl.unsetopt(pycurl.USERPWD) getattr(logging,logmethod)("%s %s", request.method, request.url) if request.client_key is not None or request.client_cert is not None: raise ValueError("Client certificate not supported with curl_httpclient") if threading.activeCount() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads # are used, signals should be disabled. This has the side effect # of disabling DNS timeouts in some environments (when libcurl is # not linked against ares), so we don't do it when there is only one # thread. Applications that use many short-lived threads may need # to set NOSIGNAL manually in a prepare_curl_callback since # there may not be any other threads running at the time we call # threading.activeCount. curl.setopt(pycurl.NOSIGNAL, 1) if request.prepare_curl_callback is not None: request.prepare_curl_callback(curl)
def decode_argument(self, value, name=None): try: return _unicode(value) except UnicodeDecodeError: raise WSException(400, 'invalid unicode argument')
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size): self.start_time = time.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.StackContext(self.cleanup): parsed = urlparse.urlsplit(_unicode(self.request.url)) if ssl is None and parsed.scheme == "https": raise ValueError("HTTPS requires either python2.6+ or " "curl_httpclient") if parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, 0, 0) af, socktype, proto, canonname, sockaddr = addrinfo[0] if parsed.scheme == "https": ssl_options = {} if request.validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED if request.ca_certs is not None: ssl_options["ca_certs"] = request.ca_certs else: ssl_options["ca_certs"] = _DEFAULT_CA_CERTS if request.client_key is not None: ssl_options["keyfile"] = request.client_key if request.client_cert is not None: ssl_options["certfile"] = request.client_cert self.stream = SSLIOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options, max_buffer_size=max_buffer_size) else: self.stream = IOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, max_buffer_size=max_buffer_size) timeout = min(request.connect_timeout, request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, self._on_timeout) self.stream.set_close_callback(self._on_close) self.stream.connect(sockaddr, functools.partial(self._on_connect, parsed))
def run(self): try: self.parsed = urlparse.urlsplit(_unicode(self.request.url)) if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") host, port = httputil.split_host_and_port(netloc) if port is None: port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] self.parsed_hostname = host # save final host for _on_connect if self.request.allow_ipv6 is False: af = socket.AF_INET else: af = socket.AF_UNSPEC ssl_options = self._get_ssl_options(self.parsed.scheme) timeout = min(self.request.connect_timeout, self.request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, stack_context.wrap( functools.partial(self._on_timeout, "while connecting"))) stream = yield self.tcp_client.connect( host, port, af=af, ssl_options=ssl_options, max_buffer_size=self.max_buffer_size) if self.final_callback is None: # final_callback is cleared if we've hit our timeout. stream.close() return self.stream = stream self.stream.set_close_callback(self.on_connection_close) self._remove_timeout() if self.final_callback is None: return if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, stack_context.wrap( functools.partial(self._on_timeout, "during request"))) if (self.request.method not in self._SUPPORTED_METHODS and not self.request.allow_nonstandard_methods): raise KeyError("unknown method %s" % self.request.method) for key in ('network_interface', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password', 'proxy_auth_mode'): if getattr(self.request, key, None): raise NotImplementedError('%s not supported' % key) if "Connection" not in self.request.headers: self.request.headers["Connection"] = "close" if "Host" not in self.request.headers: if '@' in self.parsed.netloc: self.request.headers[ "Host"] = self.parsed.netloc.rpartition('@')[-1] else: self.request.headers["Host"] = self.parsed.netloc username, password = None, None if self.parsed.username is not None: username, password = self.parsed.username, self.parsed.password elif self.request.auth_username is not None: username = self.request.auth_username password = self.request.auth_password or '' if username is not None: if self.request.auth_mode not in (None, "basic"): raise ValueError("unsupported auth_mode %s", self.request.auth_mode) auth = utf8(username) + b":" + utf8(password) self.request.headers["Authorization"] = ( b"Basic " + base64.b64encode(auth)) if self.request.user_agent: self.request.headers[ "User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: # Some HTTP methods nearly always have bodies while others # almost never do. Fail in this case unless the user has # opted out of sanity checks with allow_nonstandard_methods. body_expected = self.request.method in ("POST", "PATCH", "PUT") body_present = (self.request.body is not None or self.request.body_producer is not None) if ((body_expected and not body_present) or (body_present and not body_expected)): raise ValueError( 'Body must %sbe None for method %s (unless ' 'allow_nonstandard_methods is true)' % ('not ' if body_expected else '', self.request.method)) if self.request.expect_100_continue: self.request.headers["Expect"] = "100-continue" if self.request.body is not None: # When body_producer is used the caller is responsible for # setting Content-Length (or else chunked encoding will be used). self.request.headers["Content-Length"] = str( len(self.request.body)) if (self.request.method == "POST" and "Content-Type" not in self.request.headers): self.request.headers[ "Content-Type"] = "application/x-www-form-urlencoded" if self.request.decompress_response: self.request.headers["Accept-Encoding"] = "gzip" req_path = ((self.parsed.path or '/') + ( ('?' + self.parsed.query) if self.parsed.query else '')) self.connection = self._create_connection(stream) start_line = httputil.RequestStartLine(self.request.method, req_path, '') self.connection.write_headers(start_line, self.request.headers) if self.request.expect_100_continue: yield self.connection.read_response(self) else: yield self._write_body(True) except Exception: if not self._handle_exception(*sys.exc_info()): raise
def prepare_request(cls, request, default_host): parsed = urlparse.urlsplit(_unicode(request.url)) if (request.method not in cls._SUPPORTED_METHODS and not request.allow_nonstandard_methods): raise KeyError("unknown method %s" % request.method) request.follow_redirects = False for key in ( 'network_interface', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password', 'expect_100_continue', 'body_producer', ): if getattr(request, key, None): raise NotImplementedError('%s not supported' % key) request.headers.pop('Connection', None) if "Host" not in request.headers: if not parsed.netloc: request.headers['Host'] = default_host elif '@' in parsed.netloc: request.headers["Host"] = parsed.netloc.rpartition('@')[-1] else: request.headers["Host"] = parsed.netloc username, password = None, None if parsed.username is not None: username, password = parsed.username, parsed.password elif request.auth_username is not None: username = request.auth_username password = request.auth_password or '' if username is not None: if request.auth_mode not in (None, "basic"): raise ValueError("unsupported auth_mode %s", request.auth_mode) auth = utf8(username) + b":" + utf8(password) request.headers["Authorization"] = (b"Basic " + base64.b64encode(auth)) if request.user_agent: request.headers["User-Agent"] = request.user_agent if not request.allow_nonstandard_methods: # Some HTTP methods nearly always have bodies while others # almost never do. Fail in this case unless the user has # opted out of sanity checks with allow_nonstandard_methods. body_expected = request.method in ("POST", "PATCH", "PUT") body_present = (request.body is not None or request.body_producer is not None) if ((body_expected and not body_present) or (body_present and not body_expected)): raise ValueError( 'Body must %sbe None for method %s (unless ' 'allow_nonstandard_methods is true)' % ('not ' if body_expected else '', request.method)) if request.body is not None: # When body_producer is used the caller is responsible for # setting Content-Length (or else chunked encoding will be used). request.headers["Content-Length"] = str(len(request.body)) if (request.method == "POST" and "Content-Type" not in request.headers): request.headers[ "Content-Type"] = "application/x-www-form-urlencoded" if request.decompress_response: request.headers["Accept-Encoding"] = "gzip" request.url = ((parsed.path or '/') + (('?' + parsed.query) if parsed.query else '')) return request