def request_to_curl_string(request):
def _escape_apos(s):
return s.replace("'", "'\"'\"'")
try:
if request.body:
request.body.decode('ascii')
is_binary_data = False
except UnicodeError:
is_binary_data = True
curl_headers = HTTPHeaders(request.headers)
if request.body and 'Content-Length' not in curl_headers:
curl_headers['Content-Length'] = len(request.body)
if is_binary_data:
curl_echo_data = "echo -e {} |".format(repr(request.body))
curl_data_string = '--data-binary @-'
else:
curl_echo_data = ''
curl_data_string = "--data '{}'".format(_escape_apos(str(request.body))) if request.body else ''
return "{echo} curl -X {method} '{url}' {headers} {data}".format(
echo=curl_echo_data,
method=request.method,
url=request.url,
headers=' '.join("-H '{}: {}'".format(k, _escape_apos(str(v))) for k, v in curl_headers.items()),
data=curl_data_string
).strip()
def request_to_curl_string(request):
def _escape_apos(s):
return s.replace("'", "'\"'\"'")
try:
if request.body:
request.body.decode('ascii')
is_binary_data = False
except UnicodeError:
is_binary_data = True
curl_headers = HTTPHeaders(request.headers)
if request.body and 'Content-Length' not in curl_headers:
curl_headers['Content-Length'] = len(request.body)
if is_binary_data:
curl_echo_data = "echo -e {} |".format(repr(request.body))
curl_data_string = '--data-binary @-'
else:
curl_echo_data = ''
curl_data_string = "--data '{}'".format(_escape_apos(str(
request.body))) if request.body else ''
return "{echo} curl -X {method} '{url}' {headers} {data}".format(
echo=curl_echo_data,
method=request.method,
url=request.url,
headers=' '.join("-H '{}: {}'".format(k, _escape_apos(str(v)))
for k, v in curl_headers.items()),
data=curl_data_string).strip()
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add('Set-Cookie', 'a=b')
headers.add('Set-Cookie', 'c=d')
headers.add('Content-Type', 'text/html')
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()), sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add('Set-Cookie', 'a=b')
headers.add('Set-Cookie', 'c=d')
headers.add('Content-Type', 'text/html')
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()), sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add("Set-Cookie", "a=b")
headers.add("Set-Cookie", "c=d")
headers.add("Content-Type", "text/html")
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()), sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_pickle_roundtrip(self):
headers = HTTPHeaders()
headers.add("Set-Cookie", "a=b")
headers.add("Set-Cookie", "c=d")
headers.add("Content-Type", "text/html")
pickled = pickle.dumps(headers)
unpickled = pickle.loads(pickled)
self.assertEqual(sorted(headers.get_all()),
sorted(unpickled.get_all()))
self.assertEqual(sorted(headers.items()), sorted(unpickled.items()))
def test_proxy_pack_httprequest():
method = "POST"
uri = "/testapp/event1"
version = 'HTTP/1.0'
h = HTTPHeaders({"content-type": "text/html", "Ab": "blabla"})
body = "BODY"
host = "localhost"
req = HTTPServerRequest(method=method, uri=uri,
version=version, headers=h, connection=_FakeConnection(),
body=body, host=host)
res = pack_httprequest(req)
assert res[0] == method, "method has been parsed unproperly"
assert res[1] == uri, "uri has been parsed unproperly"
assert res[2] == "1.0", "version has been parsed unproperly %s" % res[2]
assert res[3] == h.items(), "headers has been parsed unproperly %s" % res[3]
assert res[4] == body, "body has been parsed unproperly"
def set_default_headers(self):
"""
Set any headers passed as tornado_settings['headers'].
By default sets Content-Security-Policy of frame-ancestors 'self'.
"""
# wrap in HTTPHeaders for case-insensitivity
headers = HTTPHeaders(self.settings.get('headers', {}))
headers.setdefault("X-JupyterHub-Version", __version__)
for header_name, header_content in headers.items():
self.set_header(header_name, header_content)
if 'Access-Control-Allow-Headers' not in headers:
self.set_header('Access-Control-Allow-Headers', 'accept, content-type, authorization')
if 'Content-Security-Policy' not in headers:
self.set_header('Content-Security-Policy', self.content_security_policy)
def set_default_headers(self):
"""
Set any headers passed as tornado_settings['headers'].
By default sets Content-Security-Policy of frame-ancestors 'self'.
Also responsible for setting content-type header
"""
# wrap in HTTPHeaders for case-insensitivity
headers = HTTPHeaders(self.settings.get('headers', {}))
headers.setdefault("X-JupyterHub-Version", __version__)
for header_name, header_content in headers.items():
self.set_header(header_name, header_content)
if 'Access-Control-Allow-Headers' not in headers:
self.set_header('Access-Control-Allow-Headers', 'accept, content-type, authorization')
if 'Content-Security-Policy' not in headers:
self.set_header('Content-Security-Policy', self.content_security_policy)
self.set_header('Content-Type', self.get_content_type())
def test_proxy_pack_httprequest():
method = "POST"
uri = "/testapp/event1"
version = 'HTTP/1.0'
h = HTTPHeaders({"content-type": "text/html", "Ab": "blabla"})
body = "BODY"
host = "localhost"
req = HTTPServerRequest(method=method,
uri=uri,
version=version,
headers=h,
connection=_FakeConnection(),
body=body,
host=host)
res = pack_httprequest(req)
assert res[0] == method, "method has been parsed unproperly"
assert res[1] == uri, "uri has been parsed unproperly"
assert res[2] == "1.0", "version has been parsed unproperly %s" % res[2]
assert res[3] == h.items(
), "headers has been parsed unproperly %s" % res[3]
assert res[4] == body, "body has been parsed unproperly"
class ProxyHandler(web.RequestHandler):
def initialize(self):
self.proxy_headers = HTTPHeaders()
# create a new client for each request
self.http_client = AsyncHTTPClient(max_clients=1)
self.in_request_headers = False
self.id = id(self)
self.request_data = None
def validate_request(self, request_data):
if self.request.headers.get("X-Proxy-Agent") == X_Proxy_Agent:
self.set_status(403, "recursion rejected")
return False
try:
RequstDataValidator.validate(request_data)
except ValidationError as err:
self.set_status(400, "/%s: %s" % (
"::".join(err.path), err.message
))
return False
return True
def get_post_request_data(self):
try:
request_data = json.loads(self.request.body.decode("utf-8"))
except ValueError as err:
self.set_status(400, str(err))
return
return request_data
def _set_proxy_headers(self):
for k, v in self.proxy_headers.items():
if k not in RESPONSE_EXCLUDE_HEADERS:
logger.debug(
"[%s] write header %s: %s", self.id, k, v,
)
self.set_header(k, v)
def _streaming_callback(self, chunk):
if self._finished:
return
if not self._headers_written:
self._set_proxy_headers()
self.flush()
self.in_request_headers = False
self.write(chunk)
logger.debug("[%s] chunk length %s", self.id, len(chunk))
def _header_callback(self, header_line):
if not self.in_request_headers:
start_line = parse_response_start_line(header_line)
self.set_status(start_line.code, start_line.reason)
self.in_request_headers = True
elif not HTTPHeaderEndLineRex.match(header_line):
self.proxy_headers.parse_line(header_line)
def _get_request_body(self, request_data):
post_type = request_data.get("post_type")
data = request_data.get("data")
if data is None:
return None
if post_type == "form":
body = urlencode(data or {})
elif post_type == "json":
body = json.dumps(data)
elif post_type == "string" and isinstance(data, basestring):
body = native_str(data)
else:
body = None
return body
@gen.coroutine
def _get_keystone_auth_headers(self, auth_info, validate_cert=True):
try:
response = yield self.http_client.fetch(
auth_info.get("auth_url"), method="POST",
headers={"Content-Type": "application/json"},
validate_cert=validate_cert,
body=json.dumps({
"auth": {
"passwordCredentials": {
"username": auth_info.get("user_name"),
"password": auth_info.get("password"),
},
"tenantName": auth_info.get("tenant_name"),
}
})
)
except Exception as err:
logger.info(err)
self.set_status(503, "keystone auth error")
raise gen.Return()
if response.error or response.code != 200:
logger.info("keystone auth error")
self.set_status(407, "keystone auth error")
raise gen.Return()
auth_info = json.loads(response.body.decode("utf-8"))
try:
raise gen.Return({
"X-AUTH-TOKEN": auth_info["access"]["token"]["id"],
})
except KeyError:
logger.info("keystone auth failed")
self.set_status(407, "keystone auth failed")
raise gen.Return()
def _get_proxy_request_headers(self, request_data):
headers = {
k: v for k, v in self.request.headers.items()
if k.lower() in RAW_REQUEST_ACCEPT_HEADERS
}
cookies = request_data.get("cookies")
if cookies:
headers["Cookie"] = "; ".join(
"%s=%s" % i
for i in cookies.items()
)
post_type = request_data.get("post_type")
if post_type == "form":
headers.setdefault(
"Content-Type", "application/x-www-form-urlencoded"
)
elif post_type == "json":
headers.setdefault(
"Content-Type", "application/json"
)
elif post_type == "string":
headers.setdefault(
"Content-Type", "text/plain"
)
request_headers = request_data.get("headers") or {}
for k, v in request_headers.items():
if k in REQUEST_ACCEPT_HEADERS:
headers[k] = v
elif k.startswith("X-"):
headers[k] = v
headers["X-Proxy-Agent"] = X_Proxy_Agent
return headers
@gen.coroutine
def handle_request(self, request_data):
try:
proxy_request = yield self._make_proxy_request(request_data)
if not proxy_request:
raise gen.Return()
yield self._fetch_proxy_request(proxy_request)
except RequestParamsError as err:
self.set_status(400, str(err))
except Exception as err:
logger.exception(err)
raise gen.Return()
@web.asynchronous
@gen.coroutine
def get(self):
url = self.get_query_argument("url")
logger.debug("[%s]agent get url: %s", self.id, url)
self.request_data = request_data = {"url": url}
if not self.validate_request(request_data):
raise gen.Return()
yield self.handle_request(request_data)
@web.asynchronous
@gen.coroutine
def post(self):
request_data = self.get_post_request_data()
logger.debug("[%s]agent request data: %s", self.id, request_data)
if not request_data:
raise gen.Return()
self.request_data = request_data
if not self.validate_request(request_data):
raise gen.Return()
yield self.handle_request(request_data)
def prepare_curl_callback(self, curl):
import pycurl
if (
"insecure_connection" in self.request_data and
bool(self.request_data.get("insecure_connection"))
):
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
@gen.coroutine
def _make_proxy_request(self, request_data):
timeout = float(request_data.get("timeout", DEFAULT_TIMEOUT))
validate_cert = bool(request_data.get("validate_cert") or True)
max_redirects = request_data.get("max_http_redirects") or 0
follow_redirects = max_redirects > 0 # 0 means do not follow redirects
url = request_data.get("url")
params = request_data.get("data")
post_type = request_data.get("post_type")
if params and post_type is None:
url = "%s?%s" % (url, urlencode(params))
logger.info("[%s]agent request url: %s", self.id, url)
proxy_request = HTTPRequest(
url, validate_cert=validate_cert,
headers=self._get_proxy_request_headers(request_data),
method=request_data.get("method", "GET"),
allow_nonstandard_methods=True,
connect_timeout=timeout,
request_timeout=timeout,
streaming_callback=self._streaming_callback,
header_callback=self._header_callback,
follow_redirects=follow_redirects,
max_redirects=max_redirects,
prepare_curl_callback=self.prepare_curl_callback,
)
role_name = request_data.get("role")
if role_name:
InterfaceRoleManager.set_curl_interface_role(
proxy_request, role_name,
)
keystone_auth_info = request_data.get("keystone")
if keystone_auth_info:
logger.warning(
"[%s]agent request required keystone token",
)
auth_headers = yield self._get_keystone_auth_headers(
keystone_auth_info, validate_cert=validate_cert,
)
if not auth_headers:
raise gen.Return()
proxy_request.headers.update(auth_headers)
body = self._get_request_body(request_data)
if body:
proxy_request.body = body
raise gen.Return(proxy_request)
@gen.coroutine
def _fetch_proxy_request(self, proxy_request):
self.in_request_headers = False
try:
response = yield self.http_client.fetch(proxy_request)
except HTTPError as err:
self.set_status(err.code, err.message)
raise gen.Return()
except Exception as err:
self.set_status(503, str(err))
raise gen.Return()
if response.error:
self.set_status(response.code, str(response.error))
else:
self.set_status(response.code, response.reason)
logger.info(
"[%s]agent response status: %s, reason: %s",
self.id, response.code, response.reason,
)
