def modify_and_forward(method=None,
path=None,
data_bytes=None,
headers=None,
forward_base_url=None,
listeners=None,
request_handler=None,
client_address=None,
server_address=None):
""" This is the central function that coordinates the incoming/outgoing messages
with the proxy listeners (message interceptors). """
listeners = ProxyListener.DEFAULT_LISTENERS + (listeners or [])
listeners = [lis for lis in listeners if lis]
data = data_bytes
def is_full_url(url):
return re.match(r'[a-zA-Z]+://.+', url)
if is_full_url(path):
path = path.split('://', 1)[1]
path = '/%s' % (path.split('/', 1)[1] if '/' in path else '')
proxy_url = '%s%s' % (forward_base_url, path)
for listener in listeners:
proxy_url = listener.get_forward_url(method, path, data,
headers) or proxy_url
target_url = path
if not is_full_url(target_url):
target_url = '%s%s' % (forward_base_url, target_url)
# update original "Host" header (moto s3 relies on this behavior)
if not headers.get('Host'):
headers['host'] = urlparse(target_url).netloc
headers['X-Forwarded-For'] = build_x_forwarded_for(headers, client_address,
server_address)
response = None
modified_request = None
# update listener (pre-invocation)
for listener in listeners:
listener_result = listener.forward_request(method=method,
path=path,
data=data,
headers=headers)
if isinstance(listener_result, Response):
response = listener_result
break
if isinstance(listener_result, LambdaResponse):
response = listener_result
break
if isinstance(listener_result, dict):
response = Response()
response._content = json.dumps(json_safe(listener_result))
response.headers['Content-Type'] = APPLICATION_JSON
response.status_code = 200
break
elif isinstance(listener_result, Request):
modified_request = listener_result
data = modified_request.data
headers = modified_request.headers
break
elif http2_server.get_async_generator_result(listener_result):
return listener_result
elif listener_result is not True:
# get status code from response, or use Bad Gateway status code
code = listener_result if isinstance(listener_result, int) else 503
response = Response()
response.status_code = code
response._content = ''
response.headers['Content-Length'] = '0'
append_cors_headers(response)
return response
# perform the actual invocation of the backend service
if response is None:
headers['Connection'] = headers.get('Connection') or 'close'
data_to_send = data_bytes
request_url = proxy_url
if modified_request:
if modified_request.url:
request_url = '%s%s' % (forward_base_url, modified_request.url)
data_to_send = modified_request.data
# make sure we drop "chunked" transfer encoding from the headers to be forwarded
headers.pop('Transfer-Encoding', None)
requests_method = getattr(requests, method.lower())
response = requests_method(request_url,
data=data_to_send,
headers=headers,
stream=True,
verify=False)
# prevent requests from processing response body (e.g., to pass-through gzip encoded content unmodified)
pass_raw = ((hasattr(response, '_content_consumed')
and not response._content_consumed)
or response.headers.get('content-encoding') in ['gzip'])
if pass_raw and getattr(response, 'raw', None):
new_content = response.raw.read()
if new_content:
response._content = new_content
# update listener (post-invocation)
if listeners:
update_listener = listeners[-1]
kwargs = {
'method': method,
'path': path,
'data': data_bytes,
'headers': headers,
'response': response
}
if 'request_handler' in inspect.getargspec(
update_listener.return_response)[0]:
# some listeners (e.g., sqs_listener.py) require additional details like the original
# request port, hence we pass in a reference to this request handler as well.
kwargs['request_handler'] = request_handler
updated_response = update_listener.return_response(**kwargs)
if isinstance(updated_response, Response):
response = updated_response
# allow pre-flight CORS headers by default
from localstack.services.s3.s3_listener import ProxyListenerS3
is_s3_listener = any([
isinstance(service_listener, ProxyListenerS3)
for service_listener in listeners
])
if not is_s3_listener:
append_cors_headers(response)
return response
def modify_and_forward(
method: str = None,
path: str = None,
data_bytes: bytes = None,
headers: Headers = None,
forward_base_url: str = None,
listeners: List[ProxyListener] = None,
client_address: str = None,
server_address: str = None,
):
"""This is the central function that coordinates the incoming/outgoing messages
with the proxy listeners (message interceptors)."""
from localstack.services.edge import ProxyListenerEdge
# Check origin / referer header before anything else happens.
if (not config.DISABLE_CORS_CHECKS and should_enforce_self_managed_service(
method, path, headers, data_bytes)
and not is_cors_origin_allowed(headers)):
LOG.info(
"Blocked CORS request from forbidden origin %s",
headers.get("origin") or headers.get("referer"),
)
return cors_error_response()
listeners = [lis for lis in listeners or [] if lis]
default_listeners = list(ProxyListener.DEFAULT_LISTENERS)
# ensure that MessageModifyingProxyListeners are not applied in the edge proxy request chain
# TODO: find a better approach for this!
is_edge_request = [
lis for lis in listeners if isinstance(lis, ProxyListenerEdge)
]
if is_edge_request:
default_listeners = [
lis for lis in default_listeners
if not isinstance(lis, MessageModifyingProxyListener)
]
listeners_inbound = default_listeners + listeners
listeners_outbound = listeners + default_listeners
data = data_bytes
original_request = RoutingRequest(method=method,
path=path,
data=data,
headers=headers)
def is_full_url(url):
return re.match(r"[a-zA-Z]+://.+", url)
def get_proxy_backend_url(_path, original_url=None, run_listeners=False):
if is_full_url(_path):
_path = _path.split("://", 1)[1]
_path = "/%s" % (_path.split("/", 1)[1] if "/" in _path else "")
base_url = forward_base_url or original_url
result = update_path_in_url(base_url, _path)
if run_listeners:
for listener in listeners_inbound:
result = listener.get_forward_url(method, path, data,
headers) or result
return result
target_url = path
if not is_full_url(target_url):
target_url = "%s%s" % (forward_base_url, target_url)
# update original "Host" header (moto s3 relies on this behavior)
if not headers.get("Host"):
headers["host"] = urlparse(target_url).netloc
headers["X-Forwarded-For"] = build_x_forwarded_for(headers, client_address,
server_address)
response = None
handler_chain_request = original_request.copy()
modified_request_to_backend = None
# run inbound handlers (pre-invocation)
for listener in listeners_inbound:
try:
listener_result = listener.forward_request(
method=handler_chain_request.method,
path=handler_chain_request.path,
data=handler_chain_request.data,
headers=handler_chain_request.headers,
)
except HTTPException as e:
# TODO: implement properly using exception handlers
return http_exception_to_response(e)
if isinstance(listener, MessageModifyingProxyListener):
if isinstance(listener_result, RoutingRequest):
# update the modified request details, then call next listener
handler_chain_request.method = (listener_result.method or
handler_chain_request.method)
handler_chain_request.path = listener_result.path or handler_chain_request.path
if listener_result.data is not None:
handler_chain_request.data = listener_result.data
if listener_result.headers is not None:
handler_chain_request.headers = listener_result.headers
continue
if isinstance(listener_result, Response):
response = listener_result
break
if isinstance(listener_result, LambdaResponse):
response = listener_result
break
if isinstance(listener_result, dict):
response = Response()
response._content = json.dumps(json_safe(listener_result))
response.headers["Content-Type"] = APPLICATION_JSON
response.status_code = 200
break
elif isinstance(listener_result, Request):
# TODO: unify modified_request_to_backend (requests.Request) and
# handler_chain_request (ls.routing.Request)
modified_request_to_backend = listener_result
break
elif http2_server.get_async_generator_result(listener_result):
return listener_result
elif listener_result is not True:
# get status code from response, or use Bad Gateway status code
code = listener_result if isinstance(listener_result, int) else 503
response = Response()
response.status_code = code
response._content = ""
response.headers["Content-Length"] = "0"
append_cors_headers(request_headers=headers, response=response)
return response
# perform the actual invocation of the backend service
headers_to_send = None
data_to_send = None
method_to_send = None
if response is None:
headers_to_send = handler_chain_request.headers
headers_to_send["Connection"] = headers_to_send.get(
"Connection") or "close"
data_to_send = handler_chain_request.data
method_to_send = handler_chain_request.method
request_url = get_proxy_backend_url(handler_chain_request.path,
run_listeners=True)
if modified_request_to_backend:
if modified_request_to_backend.url:
request_url = get_proxy_backend_url(
modified_request_to_backend.url, original_url=request_url)
data_to_send = modified_request_to_backend.data
if modified_request_to_backend.method:
method_to_send = modified_request_to_backend.method
# make sure we drop "chunked" transfer encoding from the headers to be forwarded
headers_to_send.pop("Transfer-Encoding", None)
response = requests.request(
method_to_send,
url=request_url,
data=data_to_send,
headers=headers_to_send,
stream=True,
verify=False,
)
# prevent requests from processing response body (e.g., to pass-through gzip encoded content
# unmodified)
not_consumed = not getattr(response, "_content_consumed", True)
pass_raw = not_consumed or response.headers.get("content-encoding") in [
"gzip"
]
if pass_raw and getattr(response, "raw", None):
new_content = response.raw.read()
if new_content:
response._content = new_content
# run outbound handlers (post-invocation)
for listener in listeners_outbound:
updated_response = listener.return_response(
method=method_to_send or handler_chain_request.method,
path=handler_chain_request.path,
data=data_to_send or handler_chain_request.data,
headers=headers_to_send or handler_chain_request.headers,
response=response,
)
message_modifier = isinstance(listener, MessageModifyingProxyListener)
if message_modifier and isinstance(updated_response, RoutingResponse):
# update the fields from updated_response in final response
response.status_code = updated_response.status_code or response.status_code
response.headers = updated_response.headers or response.headers
if isinstance(updated_response.content, (str, bytes)):
response._content = updated_response.content
if isinstance(updated_response, Response):
response = updated_response
# allow pre-flight CORS headers by default
from localstack.services.s3.s3_listener import ProxyListenerS3
is_s3_listener = any(
isinstance(service_listener, ProxyListenerS3)
for service_listener in listeners)
if not is_s3_listener:
append_cors_headers(request_headers=headers, response=response)
return response
def modify_and_forward(
method=None,
path=None,
data_bytes=None,
headers=None,
forward_base_url=None,
listeners=None,
request_handler=None,
client_address=None,
server_address=None,
):
"""This is the central function that coordinates the incoming/outgoing messages
with the proxy listeners (message interceptors)."""
# Check origin / referer header before anything else happens.
if (not config.DISABLE_CORS_CHECKS and should_enforce_self_managed_service(
method, path, headers, data_bytes)
and not is_cors_origin_allowed(headers)):
LOG.info(
"Blocked cors request from forbidden origin %s",
headers.get("origin") or headers.get("referer"),
)
return cors_error_response()
listeners = ProxyListener.DEFAULT_LISTENERS + (listeners or [])
listeners = [lis for lis in listeners if lis]
data = data_bytes
def is_full_url(url):
return re.match(r"[a-zA-Z]+://.+", url)
if is_full_url(path):
path = path.split("://", 1)[1]
path = "/%s" % (path.split("/", 1)[1] if "/" in path else "")
proxy_url = "%s%s" % (forward_base_url, path)
for listener in listeners:
proxy_url = listener.get_forward_url(method, path, data,
headers) or proxy_url
target_url = path
if not is_full_url(target_url):
target_url = "%s%s" % (forward_base_url, target_url)
# update original "Host" header (moto s3 relies on this behavior)
if not headers.get("Host"):
headers["host"] = urlparse(target_url).netloc
headers["X-Forwarded-For"] = build_x_forwarded_for(headers, client_address,
server_address)
response = None
modified_request = None
# update listener (pre-invocation)
for listener in listeners:
try:
listener_result = listener.forward_request(method=method,
path=path,
data=data,
headers=headers)
except HTTPException as e:
# TODO: implement properly using exception handlers
return http_exception_to_response(e)
if isinstance(listener_result, Response):
response = listener_result
break
if isinstance(listener_result, LambdaResponse):
response = listener_result
break
if isinstance(listener_result, dict):
response = Response()
response._content = json.dumps(json_safe(listener_result))
response.headers["Content-Type"] = APPLICATION_JSON
response.status_code = 200
break
elif isinstance(listener_result, Request):
modified_request = listener_result
data = modified_request.data
headers = modified_request.headers
break
elif http2_server.get_async_generator_result(listener_result):
return listener_result
elif listener_result is not True:
# get status code from response, or use Bad Gateway status code
code = listener_result if isinstance(listener_result, int) else 503
response = Response()
response.status_code = code
response._content = ""
response.headers["Content-Length"] = "0"
append_cors_headers(request_headers=headers, response=response)
return response
# perform the actual invocation of the backend service
if response is None:
headers["Connection"] = headers.get("Connection") or "close"
data_to_send = data_bytes
request_url = proxy_url
if modified_request:
if modified_request.url:
request_url = "%s%s" % (forward_base_url, modified_request.url)
data_to_send = modified_request.data
# make sure we drop "chunked" transfer encoding from the headers to be forwarded
headers.pop("Transfer-Encoding", None)
response = requests.request(method,
request_url,
data=data_to_send,
headers=headers,
stream=True,
verify=False)
# prevent requests from processing response body (e.g., to pass-through gzip encoded content unmodified)
pass_raw = (hasattr(response, "_content_consumed")
and not response._content_consumed
) or response.headers.get("content-encoding") in ["gzip"]
if pass_raw and getattr(response, "raw", None):
new_content = response.raw.read()
if new_content:
response._content = new_content
# update listener (post-invocation)
if listeners:
update_listener = listeners[-1]
kwargs = {
"method": method,
"path": path,
"data": data_bytes,
"headers": headers,
"response": response,
}
if "request_handler" in inspect.getfullargspec(
update_listener.return_response).args:
# some listeners (e.g., sqs_listener.py) require additional details like the original
# request port, hence we pass in a reference to this request handler as well.
kwargs["request_handler"] = request_handler
updated_response = update_listener.return_response(**kwargs)
if isinstance(updated_response, Response):
response = updated_response
# allow pre-flight CORS headers by default
from localstack.services.s3.s3_listener import ProxyListenerS3
is_s3_listener = any([
isinstance(service_listener, ProxyListenerS3)
for service_listener in listeners
])
if not is_s3_listener:
append_cors_headers(request_headers=headers, response=response)
return response