def __init__(self, url: str, code: int, result: bytes,
response_headers: HTTPHeaders,
error: Optional[BaseException]) -> None:
self._url = url
self._code = code
self._result: bytes = result
self._encoding: str = "utf-8"
self._response_headers: HTTPHeaders = response_headers
self._etag: Optional[str] = response_headers.get("etag", None)
self._error = error
self._last_modified: Optional[str] = response_headers.get(
"last-modified", None)
class _HTTPRequest(object):
def __init__(self, data):
method, url, version, headers, self._body = msgpack_unpackb(data)
if six.PY3:
method = method.decode()
url = url.decode()
version = version.decode()
headers = [(k.decode(), v.decode()) for k, v in headers]
self._headers = HTTPHeaders(headers)
self._meta = {
'method':
method,
'version':
version,
'host':
self._headers.get('Host', ''),
'remote_addr':
self._headers.get('X-Real-IP')
or self._headers.get('X-Forwarded-For', ''),
'query_string':
urlparse.urlparse(url).query,
'cookies':
dict(),
'parsed_cookies':
http_parse_cookies(self._headers),
}
args = urlparse.parse_qs(urlparse.urlparse(url).query)
self._files = dict()
parse_body_arguments(self._headers.get("Content-Type", ""), self._body,
args, self._files)
self._request = dict_list_to_single(args)
@property
def headers(self):
return self._headers
@property
def body(self):
"""Return request body"""
return self._body
@property
def meta(self):
return self._meta
@property
def request(self):
return self._request
@property
def files(self):
return self._files
class _HTTPRequest(object):
def __init__(self, request, data):
self._underlying_request = request
method, url, version, headers, self._body = msgpack_unpackb(data)
if six.PY3:
method = method.decode()
url = url.decode()
version = version.decode()
headers = [(k.decode(), v.decode()) for k, v in headers]
self._headers = HTTPHeaders(headers)
self._meta = {
'method': method,
'version': version,
'host': self._headers.get('Host', ''),
'remote_addr': self._headers.get('X-Real-IP') or self._headers.get('X-Forwarded-For', ''),
'query_string': urlparse.urlparse(url).query,
'cookies': dict(),
'parsed_cookies': http_parse_cookies(self._headers),
}
args = urlparse.parse_qs(urlparse.urlparse(url).query)
self._files = dict()
parse_body_arguments(self._headers.get("Content-Type", ""), self._body, args, self._files)
self._request = dict_list_to_single(args)
@property
def headers(self):
return self._headers
def hpack_headers(self):
return self._underlying_request.headers
@property
def body(self):
"""Return request body"""
return self._body
@property
def meta(self):
return self._meta
@property
def request(self):
return self._request
@property
def files(self):
return self._files
async def read_one(self) -> Text:
"""Read a single message"""
message = ""
headers = HTTPHeaders()
line = await convert_yielded(self._readline())
if line:
while line and line.strip():
headers.parse_line(line)
line = await convert_yielded(self._readline())
content_length = int(headers.get("content-length", "0"))
if content_length:
raw = await self._read_content(length=content_length)
if raw is not None:
message = raw.decode("utf-8").strip()
else: # pragma: no cover
self.log.warning(
"%s failed to read message of length %s",
self,
content_length,
)
return message
def make_post_request(url,
data='',
headers=None,
files=None,
content_type=None,
connect_timeout=None,
request_timeout=None,
follow_redirects=True):
if files:
body, content_type = make_mfd(data, files)
else:
body = make_body(data)
headers = HTTPHeaders() if headers is None else HTTPHeaders(headers)
if content_type is None:
content_type = headers.get('Content-Type',
'application/x-www-form-urlencoded')
headers.update({
'Content-Type': content_type,
'Content-Length': str(len(body))
})
return HTTPRequest(url=_encode(url),
body=body,
method='POST',
headers=headers,
follow_redirects=follow_redirects,
connect_timeout=connect_timeout,
request_timeout=request_timeout)
def read_tracking_id_headers(headers: HTTPHeaders, raise_error=True):
if mdc.correlation_id.get():
return
correlation_id = headers.get(HttpHeaders.X_CORRELATION_ID, None)
if not correlation_id:
correlation_id = message_utilities.get_uuid()
logger.info(
f"Request is missing {HttpHeaders.X_CORRELATION_ID} header. Assigning new value: {correlation_id}"
)
else:
if len(re.findall(UUID_PATTERN, correlation_id)) != 1:
if raise_error:
raise tornado.web.HTTPError(
status_code=400,
log_message=
f"Invalid {HttpHeaders.X_CORRELATION_ID} header. Should be an UUIDv4 matching regex '{UUID_PATTERN}'"
)
else:
correlation_id = message_utilities.get_uuid()
logger.info(
f"Invalid {HttpHeaders.X_CORRELATION_ID} header. Assigning new value: {correlation_id}"
)
mdc.correlation_id.set(correlation_id)
async def read_one(self) -> Text:
"""Read a single message"""
message = ""
headers = HTTPHeaders()
line = await convert_yielded(self._readline())
if line:
while line and line.strip():
headers.parse_line(line)
line = await convert_yielded(self._readline())
content_length = int(headers.get("content-length", "0"))
if content_length:
raw = None
retries = 5
while raw is None and retries:
try:
raw = self.stream.read(content_length)
except OSError: # pragma: no cover
raw = None
if raw is None: # pragma: no cover
self.log.warning(
"%s failed to read message of length %s",
self,
content_length,
)
await self.sleep()
retries -= 1
else:
message = raw.decode("utf-8").strip()
break
return message
def _can_keep_alive(self, start_line: httputil.RequestStartLine,
headers: httputil.HTTPHeaders) -> bool:
if self.params.no_keep_alive:
return False
connection_header = headers.get("Connection")
if connection_header is not None:
connection_header = connection_header.lower()
if start_line.version == "HTTP/1.1":
return connection_header != "close"
elif ("Content-Length" in headers
or headers.get("Transfer-Encoding", "").lower() == "chunked"
or getattr(start_line, "method", None) in ("HEAD", "GET")):
# start_line may be a request or response start line; only
# the former has a method attribute.
return connection_header == "keep-alive"
return False
def _extract_correlation_id(headers: HTTPHeaders):
correlation_id = headers.get(HttpHeaders.CORRELATION_ID, None)
if not correlation_id:
correlation_id = message_utilities.get_uuid()
logger.info(
"Didn't receive correlation id in incoming request from supplier, so have generated a new one."
)
mdc.correlation_id.set(correlation_id)
def _read_body(
self,
code: int,
headers: httputil.HTTPHeaders,
delegate: httputil.HTTPMessageDelegate,
) -> Optional[Awaitable[None]]:
if "Content-Length" in headers:
if "Transfer-Encoding" in headers:
# Response cannot contain both Content-Length and
# Transfer-Encoding headers.
# http://tools.ietf.org/html/rfc7230#section-3.3.3
raise httputil.HTTPInputError(
"Response with both Transfer-Encoding and Content-Length"
)
if "," in headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r",\s*", headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise httputil.HTTPInputError(
"Multiple unequal Content-Lengths: %r"
% headers["Content-Length"]
)
headers["Content-Length"] = pieces[0]
try:
content_length = int(headers["Content-Length"]) # type: Optional[int]
except ValueError:
# Handles non-integer Content-Length value.
raise httputil.HTTPInputError(
"Only integer Content-Length is allowed: %s"
% headers["Content-Length"]
)
if cast(int, content_length) > self._max_body_size:
raise httputil.HTTPInputError("Content-Length too long")
else:
content_length = None
if code == 204:
# This response code is not allowed to have a non-empty body,
# and has an implicit length of zero instead of read-until-close.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
if "Transfer-Encoding" in headers or content_length not in (None, 0):
raise httputil.HTTPInputError(
"Response with code %d should not have body" % code
)
content_length = 0
if content_length is not None:
return self._read_fixed_body(content_length, delegate)
if headers.get("Transfer-Encoding", "").lower() == "chunked":
return self._read_chunked_body(delegate)
if self.is_client:
return self._read_body_until_close(delegate)
return None
def _read_body(
self,
code: int,
headers: httputil.HTTPHeaders,
delegate: httputil.HTTPMessageDelegate,
) -> Optional[Awaitable[None]]:
if "Content-Length" in headers:
if "Transfer-Encoding" in headers:
# Response cannot contain both Content-Length and
# Transfer-Encoding headers.
# http://tools.ietf.org/html/rfc7230#section-3.3.3
raise httputil.HTTPInputError(
"Response with both Transfer-Encoding and Content-Length"
)
if "," in headers["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r",\s*", headers["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise httputil.HTTPInputError(
"Multiple unequal Content-Lengths: %r"
% headers["Content-Length"]
)
headers["Content-Length"] = pieces[0]
try:
content_length = int(headers["Content-Length"]) # type: Optional[int]
except ValueError:
# Handles non-integer Content-Length value.
raise httputil.HTTPInputError(
"Only integer Content-Length is allowed: %s"
% headers["Content-Length"]
)
if cast(int, content_length) > self._max_body_size:
raise httputil.HTTPInputError("Content-Length too long")
else:
content_length = None
if code == 204:
# This response code is not allowed to have a non-empty body,
# and has an implicit length of zero instead of read-until-close.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
if "Transfer-Encoding" in headers or content_length not in (None, 0):
raise httputil.HTTPInputError(
"Response with code %d should not have body" % code
)
content_length = 0
if content_length is not None:
return self._read_fixed_body(content_length, delegate)
if headers.get("Transfer-Encoding", "").lower() == "chunked":
return self._read_chunked_body(delegate)
if self.is_client:
return self._read_body_until_close(delegate)
return None
def _can_keep_alive(
self, start_line: httputil.RequestStartLine, headers: httputil.HTTPHeaders
) -> bool:
if self.params.no_keep_alive:
return False
connection_header = headers.get("Connection")
if connection_header is not None:
connection_header = connection_header.lower()
if start_line.version == "HTTP/1.1":
return connection_header != "close"
elif (
"Content-Length" in headers
or headers.get("Transfer-Encoding", "").lower() == "chunked"
or getattr(start_line, "method", None) in ("HEAD", "GET")
):
# start_line may be a request or response start line; only
# the former has a method attribute.
return connection_header == "keep-alive"
return False
def test_urllib2(scheme, root_span, install_hooks):
request = urllib2.Request('%s://localhost:9777/proxy' % scheme,
headers={'Remote-LOC': 'New New York',
'Remote-Op': 'antiquing'})
class Response(object):
def __init__(self):
self.code = 200
self.msg = ''
def info(self):
return None
if root_span:
root_span = mock.MagicMock()
root_span.context = mock.MagicMock()
root_span.finish = mock.MagicMock()
root_span.__exit__ = mock.MagicMock()
else:
root_span = None
span = mock.MagicMock()
span.set_tag = mock.MagicMock()
span.finish = mock.MagicMock()
def inject(span_context, format, carrier):
carrier['TRACE-ID'] = '123'
p_do_open = mock.patch('urllib2.AbstractHTTPHandler.do_open',
return_value=Response())
p_start_span = mock.patch.object(opentracing.tracer, 'start_span',
return_value=span)
p_inject = mock.patch.object(opentracing.tracer, 'inject',
side_effect=inject)
p_current_span = span_in_context(span=root_span)
with p_do_open, p_start_span as start_call, p_inject, p_current_span:
resp = urllib2.urlopen(request)
expected_references = root_span.context if root_span else None
start_call.assert_called_once_with(
operation_name='GET:antiquing',
child_of=expected_references,
tags=None,
)
assert resp is not None
span.set_tag.assert_any_call('span.kind', 'client')
assert span.__enter__.call_count == 1
assert span.__exit__.call_count == 1, 'ensure finish() was called'
if root_span:
assert root_span.__exit__.call_count == 0, 'do not finish root span'
# verify trace-id was correctly injected into headers
norm_headers = HTTPHeaders(request.headers)
assert norm_headers.get('trace-id') == '123'
def _apply_xheaders(self, headers: httputil.HTTPHeaders) -> None:
"""Rewrite the ``remote_ip`` and ``protocol`` fields."""
# Squid uses X-Forwarded-For, others use X-Real-Ip
ip = headers.get("X-Forwarded-For", self.remote_ip)
# Skip trusted downstream hosts in X-Forwarded-For list
for ip in (cand.strip() for cand in reversed(ip.split(","))):
if ip not in self.trusted_downstream:
break
ip = headers.get("X-Real-Ip", ip)
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
proto_header = headers.get(
"X-Scheme", headers.get("X-Forwarded-Proto", self.protocol))
if proto_header:
# use only the last proto entry if there is more than one
# TODO: support trusting mutiple layers of proxied protocol
proto_header = proto_header.split(",")[-1].strip()
if proto_header in ("http", "https"):
self.protocol = proto_header
def headers_received(self, start_line: Union[httputil.RequestStartLine,
httputil.ResponseStartLine],
headers: httputil.HTTPHeaders) -> Optional[Awaitable[None]]:
if headers.get("Content-Encoding") == "gzip":
self._decompressor = GzipDecompressor()
# Downstream delegates will only see uncompressed data,
# so rename the content-encoding header.
# (but note that curl_httpclient doesn't do this).
headers.add("X-Consumed-Content-Encoding",
headers["Content-Encoding"])
del headers["Content-Encoding"]
return self._delegate.headers_received(start_line, headers)
def test_urllib2(urllibver, scheme, root_span, install_hooks, tracer):
module = install_hooks
if module is None:
pytest.skip('Skipping %s on Py3' % urllibver)
class Response(object):
def __init__(self):
self.code = 200
self.msg = ''
def info(self):
return None
if root_span:
root_span = tracer.start_span('root-span')
else:
root_span = None
# ideally we should have started a test server and tested with real HTTP
# request, but doing that for https is more difficult, so we mock the
# request sending part.
if urllibver == 'urllib2':
p_do_open = mock.patch(
'urllib2.AbstractHTTPHandler.do_open', return_value=Response()
)
else:
cls = module.AbstractHTTPHandler
p_do_open = mock._patch_object(
cls, 'do_open', return_value=Response()
)
with p_do_open, span_in_context(span=root_span):
request = module.Request(
'%s://localhost:9777/proxy' % scheme,
headers={
'Remote-LOC': 'New New York',
'Remote-Op': 'antiquing'
})
resp = module.urlopen(request)
assert resp.code == 200
assert len(tracer.recorder.get_spans()) == 1
span = tracer.recorder.get_spans()[0]
assert span.tags.get('span.kind') == 'client'
# verify trace-id was correctly injected into headers
# we wrap the headers to avoid having to deal with upper/lower case
norm_headers = HTTPHeaders(request.headers)
trace_id_header = norm_headers.get('ot-tracer-traceid')
assert trace_id_header == '%x' % span.context.trace_id
def _apply_xheaders(self, headers: httputil.HTTPHeaders) -> None:
"""Rewrite the ``remote_ip`` and ``protocol`` fields."""
# Squid uses X-Forwarded-For, others use X-Real-Ip
ip = headers.get("X-Forwarded-For", self.remote_ip)
# Skip trusted downstream hosts in X-Forwarded-For list
for ip in (cand.strip() for cand in reversed(ip.split(","))):
if ip not in self.trusted_downstream:
break
ip = headers.get("X-Real-Ip", ip)
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
proto_header = headers.get(
"X-Scheme", headers.get("X-Forwarded-Proto", self.protocol)
)
if proto_header:
# use only the last proto entry if there is more than one
# TODO: support trusting mutiple layers of proxied protocol
proto_header = proto_header.split(",")[-1].strip()
if proto_header in ("http", "https"):
self.protocol = proto_header
def headers_received(
self,
start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine],
headers: httputil.HTTPHeaders,
) -> Optional[Awaitable[None]]:
if headers.get("Content-Encoding") == "gzip":
self._decompressor = GzipDecompressor()
# Downstream delegates will only see uncompressed data,
# so rename the content-encoding header.
# (but note that curl_httpclient doesn't do this).
headers.add("X-Consumed-Content-Encoding", headers["Content-Encoding"])
del headers["Content-Encoding"]
return self._delegate.headers_received(start_line, headers)
def get_valid_accept_type(headers: HTTPHeaders):
accept_types = headers.get(HttpHeaders.ACCEPT,
APPLICATION_FHIR_JSON).lower()
accept_types = accept_types.split(",")
accept_types = list(map(lambda value: value.strip(), accept_types))
if ANY in accept_types or APPLICATION_FHIR_JSON in accept_types:
return APPLICATION_FHIR_JSON
elif APPLICATION_JSON in accept_types:
return APPLICATION_JSON
else:
logger.info("Invalid Accept header in request")
raise tornado.web.HTTPError(
status_code=406,
log_message=
f'Invalid Accept header in request, only: {APPLICATION_JSON} or {APPLICATION_FHIR_JSON} are allowed'
)
def make_post_request(url, data='', headers=None, files=None, content_type=None,
connect_timeout=None, request_timeout=None, follow_redirects=True):
if files:
body, content_type = make_mfd(data, files)
else:
body = make_body(data)
headers = HTTPHeaders() if headers is None else HTTPHeaders(headers)
if content_type is None:
content_type = headers.get('Content-Type', 'application/x-www-form-urlencoded')
headers.update({'Content-Type': content_type, 'Content-Length': str(len(body))})
return HTTPRequest(
url=url,
body=body,
method='POST',
headers=headers,
follow_redirects=follow_redirects,
connect_timeout=connect_timeout,
request_timeout=request_timeout
)
async def _build_auth_info(headers: HTTPHeaders) -> Dict[str, Any]:
"""Construct the authentication information for a user.
Retrieve the token from the headers, use that token to retrieve the
metadata for the token, and use that data to build an auth info dict
in the format expected by JupyterHub. This is in a separate method so
that it can be unit-tested.
"""
token = headers.get("X-Auth-Request-Token")
if not token:
raise web.HTTPError(401, "No request token")
config = NubladoConfig()
if not config.gafaelfawr_token:
raise web.HTTPError(500, "gafaelfawr_token not set in configuration")
if not config.base_url:
raise web.HTTPError(500, "base_url not set in configuration")
# Retrieve the token metadata.
api_url = url_path_join(config.base_url, "/auth/api/v1/user-info")
session = await get_session()
resp = await session.get(api_url,
headers={"Authorization": f"bearer {token}"})
if resp.status != 200:
raise web.HTTPError(500, "Cannot reach token analysis API")
try:
auth_state = await resp.json()
except Exception:
raise web.HTTPError(500, "Cannot get information for token")
if "username" not in auth_state or "uid" not in auth_state:
raise web.HTTPError(403, "Request token is invalid")
auth_state["token"] = token
if "groups" not in auth_state:
auth_state["groups"] = []
return {
"name": auth_state["username"],
"auth_state": auth_state,
}
class ProxyHandler(tornado.web.StaticFileHandler):
CHUNK_SIZE = 64 * 1024
SUPPORTED_METHODS = ['GET', 'CONNECT']
def initialize(self, path, default_filename=None):
self.cache_dir = path
self.url_transpose = self.application.url_transpose
tornado.web.StaticFileHandler.initialize(self, str(self.cache_dir))
def data_received(self, chunk):
raise NotImplementedError()
def prepare(self):
self.cacheable_exts = ('.rpm', '.img', '.sqlite.bz2', '.sqlite.gz', '.xml', '.xml.gz', '.qcow2', '.raw.xz',
'.iso', 'filelist.gz', 'vmlinuz')
self.cacheable = False
self.cache_used = False
self.cache_file = None
self.cache_fd = None
self.cache_url = False
self.req_code = None
self.req_path = None
self.req_headers = None
def is_cacheable(self, path):
return path.endswith(self.cacheable_exts)
@tornado.gen.coroutine
@tornado.web.asynchronous
def get(self, path, include_body=True):
self.req_path = path
app_log.info('process %s', path)
url = urlsplit(path)
self.cache_url = path.replace(url[0] + '://', '')
self.cacheable = self.is_cacheable(url.path)
app_log.debug('is cacheable %r', self.cacheable)
if self.cacheable:
cache_file = self.url_transpose(path)
if not cache_file:
netloc = [x for x in reversed(url.netloc.split('.'))]
self.cache_file = self.cache_dir / '.'.join(netloc) / url.path[1:]
else:
self.cache_file = self.cache_dir / cache_file
else:
uri = self.request.uri.encode()
cache_id = hashlib.sha1(uri).hexdigest()
cache_path = self.cache_dir / '~' / cache_id[:2]
cache_info = cache_path / (cache_id + '-url.txt')
if not cache_info.exists():
if not cache_info.parent.exists():
cache_info.parent.mkdir(parents=True)
with cache_info.open('w') as f:
f.write(uri.decode())
self.cache_file = cache_path / (cache_id + '-data.txt')
cache_time = None
if self.cache_file.exists():
self.cache_file = self.cache_file.resolve()
cache_time = self.cache_file.stat().st_mtime
lifetime = time() - int(self.settings['cache']['lifetime']) * 60 * 60
app_log.debug('cache time is %r lifetime is %r', cache_time, lifetime)
if cache_time > lifetime:
app_log.info('found %s', self.cache_file)
cache_url = self.cache_file.relative_to(self.cache_dir).as_posix()
return tornado.web.StaticFileHandler.get(self, cache_url)
app_log.info('%s lifetime exceeded', self.cache_file)
args = {k: v[0] for k, v in self.request.arguments.items()}
app_log.info('fetch %s', self.request.uri)
if 'Range' in self.request.headers:
del self.request.headers['Range']
self.client = AsyncHTTPClient()
self.client.fetch(self.request.uri,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
if_modified_since=cache_time,
allow_nonstandard_methods=True,
connect_timeout=int(self.settings['proxy']['timeout']),
request_timeout=2 ** 31 - 1,
header_callback=self.process_header,
streaming_callback=self.process_body,
callback=self.process_finish)
def process_header(self, line):
header = line.strip()
app_log.debug('response header %s', header)
if header:
if self.req_headers is None:
self.req_headers = HTTPHeaders()
_, status, _ = header.split(' ', 2)
status = int(status)
if status == 599:
# network error but cache file exists
if self.cache_file.exists():
status = 200
elif status == 304:
status = 200
elif status == 200:
app_log.debug('prepare temp file for %s', self.req_path)
self.cache_fd = NamedTemporaryFile(dir=str(self.cache_dir), delete=False)
self.set_status(status)
else:
self.req_headers.parse_line(line)
return
for header in ('Date', 'Cache-Control', 'Server', 'Content-Type', 'Location'):
val = self.req_headers.get(header)
if val:
self.set_header(header, val)
if 'content-encoding' not in self.req_headers:
val = self.req_headers.get('Content-Length')
if val:
self.set_header('Content-Length', val)
self.flush()
def process_body(self, chunk):
if self._finished:
return
if self.cache_fd is not None:
self.cache_fd.write(chunk)
self.write(chunk)
self.flush()
def process_finish(self, response):
app_log.debug('process finish %s', self.req_path)
if self._finished or self.cache_used:
app_log.debug('skip process finish')
return
app_log.info('code %s fo %s', response.code, self.request.uri)
if response.code in (599, 304):
if self.cache_file.exists():
if response.code == 304:
self.cache_file.touch()
app_log.info('use %s', self.cache_file)
self.cache_fd = self.cache_file.open('rb')
self.process_file()
return
elif 200 <= response.code < 300:
if self.cache_fd is not None:
self.cache_fd.close()
if self.cache_file.exists():
self.cache_file.unlink()
elif not self.cache_file.parent.exists():
self.cache_file.parent.mkdir(parents=True)
temp_file = Path(self.cache_dir) / self.cache_fd.name
temp_file.rename(self.cache_file)
app_log.info('saved %s', self.cache_file)
self.cache_fd = None
self.finish()
def process_file(self):
chunk = self.cache_fd.read(self.CHUNK_SIZE)
if chunk:
self.write(chunk)
self.flush(callback=self.process_file)
return
self.cache_fd.close()
self.cache_fd = None
app_log.debug('process file %s finish', self.cache_file)
self.finish()
def compute_etag(self):
if self.cache_file is None or not self.cache_file.exists():
return None
if not hasattr(self, 'absolute_path'):
self.absolute_path = str(self.cache_file.absolute())
return tornado.web.StaticFileHandler.compute_etag(self)
def on_finish(self):
app_log.debug('on finish')
# sometimes, prepare is not called.
if not hasattr(self, 'cache_fd') or self.cache_fd is None:
return
self.cache_fd.close()
@tornado.web.asynchronous
def connect(self, path):
app_log.info('CONNECT to %s', self.request.uri)
host, port = self.request.uri.split(':')
client = self.request.connection.stream
def read_from_client(data):
# app_log.debug('read from client\n%s', data)
upstream.write(data)
def read_from_upstream(data):
# app_log.debug('read from upstream\n%s', data)
client.write(data)
def client_close(data=None):
# app_log.debug('client close\n%s', data)
if upstream.closed():
return
if data:
upstream.write(data)
upstream.close()
def upstream_close(data=None):
# app_log.debug('upstream close\n%s', data)
if client.closed():
return
if data:
client.write(data)
client.close()
def start_tunnel():
app_log.debug('start connect tunnel')
client.read_until_close(client_close, read_from_client)
upstream.read_until_close(upstream_close, read_from_upstream)
client.write(b'HTTP/1.0 200 Connection established\r\n\r\n')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
upstream = tornado.iostream.IOStream(s)
app_log.debug('connect to upstream')
upstream.connect((host, int(port)), start_tunnel)
class GatewayApi(tornado.websocket.WebSocketHandler):
# 针对websocket请求, 接收消息
async def on_message(self, message):
if self._up_websocket is not None:
# 将接收到的消息发送给upstream websocket
await self._up_websocket.send(message)
# 针对websocket请求断开连接时处理, 因为tornado的实现, 这个方法不能声明为异步的
def on_ws_connection_close(self,
close_code: int = None,
close_reason: str = None):
if self._up_websocket is not None and self._up_websocket.closed is False:
import tornado.ioloop
# 因为up_websocket.close是异步的，需要启动一个task来执行
io_loop = tornado.ioloop.IOLoop.current()
io_loop.spawn_callback(self._up_websocket.close)
super().on_ws_connection_close()
async def get(self, *args, **kwargs):
# 判断请求是否是通过HTTP进行Upgrade的websocket
if self.request.headers.get("Upgrade", "").lower() == "websocket":
# 调用相应的app_core来获取代理的upstream上游的websocket connection, 建立上游连接
self._up_websocket = await g.app_core.get_up_websocket(
self.request, self)
# 调用父类的get的方法处理websocket请求, 产生响应
await super().get(*args, **kwargs)
else:
# 其他HTTP web请求按照正常逻辑处理
output = {'result': {}, 'finish': False}
thread = threading.Thread(target=gateway_service,
args=(self.request, output))
thread.setDaemon(True)
thread.start()
while not output['finish']:
await asyncio.sleep(0.01)
result = output['result']
if result['status'] == 'ok':
new_access_token = result['value'].get(
'new_access_token') # 用 get 而不是 [] 来取值，因为可能不存在
new_refresh_token = result['value'].get('new_refresh_token')
response = result['value']['response']
if new_access_token is not None and new_refresh_token is not None:
self.set_cookie('access_token', new_access_token)
self.set_cookie('refresh_token', new_refresh_token)
self.set_status(response.status_code)
self._headers = HTTPHeaders(response.headers) # 注意：需要强制类型转换
if self._headers.get('Content-Type') == 'gzip':
try:
self._headers.pop('Content-Type')
self._headers.pop('Content-Length')
except:
pass
if self.request.headers.get('Origin'):
self.set_header('Access-Control-Allow-Credentials', 'true')
self.set_header('Access-Control-Allow-Origin',
self.request.headers.get('Origin'))
if self._status_code in (
204, 304) or 100 <= self._status_code < 200:
# 这些状态下response中不能有body，所以不应该write
return
# 判断响应是否是chunk传输
transfer_encoding_header = response.headers.get(
"Transfer-Encoding")
if transfer_encoding_header is None or transfer_encoding_header != "chunked":
# 不是的话按照正常逻辑处理
self.write(response.content)
else:
# chunk传输，chunk处理
self._headers.pop("Transfer-Encoding") # 注意一定要pop掉
raw_content = response.raw
chunk_size = 1024
while True:
# 分块读取响应, 并flush到客户端
chunk = raw_content.read(chunk_size)
self.write(chunk)
await self.flush()
if len(chunk) < chunk_size:
break
else:
self.set_status(500)
self.write(result['value'])
async def post(self, *args, **kwargs):
output = {'result': {}, 'finish': False}
thread = threading.Thread(target=gateway_service,
args=(self.request, output))
thread.setDaemon(True)
thread.start()
while not output['finish']:
await asyncio.sleep(0.01)
result = output['result']
if result['status'] == 'ok':
new_access_token = result['value'].get(
'new_access_token') # 用 get 而不是 [] 来取值，因为可能不存在
new_refresh_token = result['value'].get('new_refresh_token')
response = result['value']['response']
if new_access_token is not None and new_refresh_token is not None:
self.set_cookie('access_token', new_access_token)
self.set_cookie('refresh_token', new_refresh_token)
self.set_status(response.status_code)
self._headers = HTTPHeaders(response.headers) # 注意：需要强制类型转换
if self.request.headers.get('Origin'):
self.set_header('Access-Control-Allow-Credentials', 'true')
self.set_header('Access-Control-Allow-Origin',
self.request.headers.get('Origin'))
if self._status_code in (204,
304) or 100 <= self._status_code < 200:
# 这些状态下response中不能有body，所以不应该write
return
self.write(response.content)
else:
self.set_status(500)
self.write(result['value'])
async def options(self, *args, **kwargs):
# 允许跨域
self.set_status(204)
self.set_header('Access-Control-Allow-Credentials', 'true')
self.set_header('Access-Control-Allow-Origin',
self.request.headers.get('Origin'))
self.set_header("Access-Control-Allow-Headers", "content-type")
self.set_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
def _extract_inbound_message_id(headers: HTTPHeaders):
inbound_message_id = headers.get(HttpHeaders.INBOUND_MESSAGE_ID, None)
if inbound_message_id:
mdc.interaction_id.set(inbound_message_id)
logger.info('Found inbound message id on incoming request.')
def _extract_interaction_id(headers: HTTPHeaders):
interaction_id = headers.get(HttpHeaders.INTERACTION_ID, None)
if interaction_id:
mdc.interaction_id.set(interaction_id)
logger.info('Found interaction id on incoming request.')
def _extract_message_id(headers: HTTPHeaders):
message_id = headers.get(HttpHeaders.MESSAGE_ID, None)
if message_id:
mdc.message_id.set(message_id)
logger.info('Found message id on incoming request.')
class BalancedHttpRequest:
def __init__(self,
host: str,
upstream: Upstream,
source_app: str,
uri: str,
name: str,
method='GET',
data=None,
headers=None,
files=None,
content_type=None,
connect_timeout=None,
request_timeout=None,
max_timeout_tries=None,
follow_redirects=True,
idempotent=True):
self.source_app = source_app
self.uri = uri if uri.startswith('/') else '/' + uri
self.upstream = upstream
self.name = name
self.method = method
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.idempotent = idempotent
self.body = None
self.last_request = None
if request_timeout is not None and max_timeout_tries is None:
max_timeout_tries = options.http_client_default_max_timeout_tries
if self.connect_timeout is None:
self.connect_timeout = self.upstream.connect_timeout
if self.request_timeout is None:
self.request_timeout = self.upstream.request_timeout
if max_timeout_tries is None:
max_timeout_tries = self.upstream.max_timeout_tries
self.session_required = self.upstream.session_required
self.connect_timeout *= options.timeout_multiplier
self.request_timeout *= options.timeout_multiplier
self.headers = HTTPHeaders() if headers is None else HTTPHeaders(
headers)
if self.source_app and not self.headers.get(USER_AGENT_HEADER):
self.headers[USER_AGENT_HEADER] = self.source_app
if self.method == 'POST':
if files:
self.body, content_type = make_mfd(data, files)
else:
self.body = make_body(data)
if content_type is None:
content_type = self.headers.get(
'Content-Type', 'application/x-www-form-urlencoded')
self.headers['Content-Length'] = str(len(self.body))
elif self.method == 'PUT':
self.body = make_body(data)
else:
self.uri = make_url(self.uri, **({} if data is None else data))
if content_type is not None:
self.headers['Content-Type'] = content_type
self.request_time_left = self.request_timeout * max_timeout_tries
self.tries = OrderedDict()
self.current_host = host.rstrip('/')
self.current_server_index = None
self.current_rack = None
self.current_datacenter = None
def make_request(self):
if self.upstream.balanced:
index, host, rack, datacenter = self.upstream.borrow_server(
self.tries.keys() if self.tries else None)
self.current_server_index = index
self.current_host = host
self.current_rack = rack
self.current_datacenter = datacenter
request = HTTPRequest(
url=(self.get_calling_address()) + self.uri,
body=self.body,
method=self.method,
headers=self.headers,
follow_redirects=self.follow_redirects,
connect_timeout=self.connect_timeout,
request_timeout=self.request_timeout,
)
request.upstream_name = self.upstream.name if self.upstream.balanced else self.current_host
request.upstream_datacenter = self.current_datacenter
if options.http_proxy_host is not None:
request.proxy_host = options.http_proxy_host
request.proxy_port = options.http_proxy_port
self.last_request = request
return self.last_request
def backend_available(self):
return self.current_host is not None
def get_calling_address(self):
return self.current_host if self.backend_available(
) else self.upstream.name
def get_host(self):
return self.upstream.name if self.upstream.balanced else self.current_host
def check_retry(self, response):
self.request_time_left -= response.request_time
if self.upstream.balanced:
do_retry = self.upstream.retry_policy.check_retry(
response, self.idempotent)
if self.current_server_index is not None:
self.upstream.return_server(self.current_server_index)
else:
do_retry = False
do_retry = do_retry and self.upstream.max_tries > len(
self.tries) and self.request_time_left > 0
return do_retry
def pop_last_request(self):
request = self.last_request
self.last_request = None
return request
def register_try(self, response):
index = self.current_server_index if self.current_server_index else len(
self.tries)
self.tries[index] = ResponseData(response.code, str(response.error))
@staticmethod
def get_url(request):
return f'http://{request.get_host()}{request.uri}'
@staticmethod
def get_trace(request):
def _get_server_address(index):
if request.upstream.balanced:
if index < len(request.upstream.servers
) and request.upstream.servers[index]:
return request.upstream.servers[index].address
return f'no_idx_{index}_in_upstream'
return request.get_calling_address()
return ' -> '.join([
f'{_get_server_address(index)}~{data.responseCode}~{data.msg}'
for index, data in request.tries.items()
])
def _groups_from_headers(self, headers: HTTPHeaders) -> List[str]:
"""Parse the header containing the user's groups and return them as a list"""
return [
group.strip()
for group in headers.get(self.user_groups_header, "").split(",")
]
