def modify_and_forward(method=None, path=None, data_bytes=None, headers=None, forward_base_url=None, listeners=None, request_handler=None, client_address=None, server_address=None): """ This is the central function that coordinates the incoming/outgoing messages with the proxy listeners (message interceptors). """ listeners = ProxyListener.DEFAULT_LISTENERS + (listeners or []) listeners = [lis for lis in listeners if lis] data = data_bytes def is_full_url(url): return re.match(r'[a-zA-Z]+://.+', url) if is_full_url(path): path = path.split('://', 1)[1] path = '/%s' % (path.split('/', 1)[1] if '/' in path else '') proxy_url = '%s%s' % (forward_base_url, path) for listener in listeners: proxy_url = listener.get_forward_url(method, path, data, headers) or proxy_url target_url = path if not is_full_url(target_url): target_url = '%s%s' % (forward_base_url, target_url) # update original "Host" header (moto s3 relies on this behavior) if not headers.get('Host'): headers['host'] = urlparse(target_url).netloc headers['X-Forwarded-For'] = build_x_forwarded_for(headers, client_address, server_address) response = None modified_request = None # update listener (pre-invocation) for listener in listeners: listener_result = listener.forward_request(method=method, path=path, data=data, headers=headers) if isinstance(listener_result, Response): response = listener_result break if isinstance(listener_result, LambdaResponse): response = listener_result break if isinstance(listener_result, dict): response = Response() response._content = json.dumps(json_safe(listener_result)) response.headers['Content-Type'] = APPLICATION_JSON response.status_code = 200 break elif isinstance(listener_result, Request): modified_request = listener_result data = modified_request.data headers = modified_request.headers break elif http2_server.get_async_generator_result(listener_result): return listener_result elif listener_result is not True: # get status code from response, or use Bad Gateway status code code = listener_result if isinstance(listener_result, int) else 503 response = Response() response.status_code = code response._content = '' response.headers['Content-Length'] = '0' append_cors_headers(response) return response # perform the actual invocation of the backend service if response is None: headers['Connection'] = headers.get('Connection') or 'close' data_to_send = data_bytes request_url = proxy_url if modified_request: if modified_request.url: request_url = '%s%s' % (forward_base_url, modified_request.url) data_to_send = modified_request.data # make sure we drop "chunked" transfer encoding from the headers to be forwarded headers.pop('Transfer-Encoding', None) requests_method = getattr(requests, method.lower()) response = requests_method(request_url, data=data_to_send, headers=headers, stream=True, verify=False) # prevent requests from processing response body (e.g., to pass-through gzip encoded content unmodified) pass_raw = ((hasattr(response, '_content_consumed') and not response._content_consumed) or response.headers.get('content-encoding') in ['gzip']) if pass_raw and getattr(response, 'raw', None): new_content = response.raw.read() if new_content: response._content = new_content # update listener (post-invocation) if listeners: update_listener = listeners[-1] kwargs = { 'method': method, 'path': path, 'data': data_bytes, 'headers': headers, 'response': response } if 'request_handler' in inspect.getargspec( update_listener.return_response)[0]: # some listeners (e.g., sqs_listener.py) require additional details like the original # request port, hence we pass in a reference to this request handler as well. kwargs['request_handler'] = request_handler updated_response = update_listener.return_response(**kwargs) if isinstance(updated_response, Response): response = updated_response # allow pre-flight CORS headers by default from localstack.services.s3.s3_listener import ProxyListenerS3 is_s3_listener = any([ isinstance(service_listener, ProxyListenerS3) for service_listener in listeners ]) if not is_s3_listener: append_cors_headers(response) return response
def modify_and_forward( method: str = None, path: str = None, data_bytes: bytes = None, headers: Headers = None, forward_base_url: str = None, listeners: List[ProxyListener] = None, client_address: str = None, server_address: str = None, ): """This is the central function that coordinates the incoming/outgoing messages with the proxy listeners (message interceptors).""" from localstack.services.edge import ProxyListenerEdge # Check origin / referer header before anything else happens. if (not config.DISABLE_CORS_CHECKS and should_enforce_self_managed_service( method, path, headers, data_bytes) and not is_cors_origin_allowed(headers)): LOG.info( "Blocked CORS request from forbidden origin %s", headers.get("origin") or headers.get("referer"), ) return cors_error_response() listeners = [lis for lis in listeners or [] if lis] default_listeners = list(ProxyListener.DEFAULT_LISTENERS) # ensure that MessageModifyingProxyListeners are not applied in the edge proxy request chain # TODO: find a better approach for this! is_edge_request = [ lis for lis in listeners if isinstance(lis, ProxyListenerEdge) ] if is_edge_request: default_listeners = [ lis for lis in default_listeners if not isinstance(lis, MessageModifyingProxyListener) ] listeners_inbound = default_listeners + listeners listeners_outbound = listeners + default_listeners data = data_bytes original_request = RoutingRequest(method=method, path=path, data=data, headers=headers) def is_full_url(url): return re.match(r"[a-zA-Z]+://.+", url) def get_proxy_backend_url(_path, original_url=None, run_listeners=False): if is_full_url(_path): _path = _path.split("://", 1)[1] _path = "/%s" % (_path.split("/", 1)[1] if "/" in _path else "") base_url = forward_base_url or original_url result = update_path_in_url(base_url, _path) if run_listeners: for listener in listeners_inbound: result = listener.get_forward_url(method, path, data, headers) or result return result target_url = path if not is_full_url(target_url): target_url = "%s%s" % (forward_base_url, target_url) # update original "Host" header (moto s3 relies on this behavior) if not headers.get("Host"): headers["host"] = urlparse(target_url).netloc headers["X-Forwarded-For"] = build_x_forwarded_for(headers, client_address, server_address) response = None handler_chain_request = original_request.copy() modified_request_to_backend = None # run inbound handlers (pre-invocation) for listener in listeners_inbound: try: listener_result = listener.forward_request( method=handler_chain_request.method, path=handler_chain_request.path, data=handler_chain_request.data, headers=handler_chain_request.headers, ) except HTTPException as e: # TODO: implement properly using exception handlers return http_exception_to_response(e) if isinstance(listener, MessageModifyingProxyListener): if isinstance(listener_result, RoutingRequest): # update the modified request details, then call next listener handler_chain_request.method = (listener_result.method or handler_chain_request.method) handler_chain_request.path = listener_result.path or handler_chain_request.path if listener_result.data is not None: handler_chain_request.data = listener_result.data if listener_result.headers is not None: handler_chain_request.headers = listener_result.headers continue if isinstance(listener_result, Response): response = listener_result break if isinstance(listener_result, LambdaResponse): response = listener_result break if isinstance(listener_result, dict): response = Response() response._content = json.dumps(json_safe(listener_result)) response.headers["Content-Type"] = APPLICATION_JSON response.status_code = 200 break elif isinstance(listener_result, Request): # TODO: unify modified_request_to_backend (requests.Request) and # handler_chain_request (ls.routing.Request) modified_request_to_backend = listener_result break elif http2_server.get_async_generator_result(listener_result): return listener_result elif listener_result is not True: # get status code from response, or use Bad Gateway status code code = listener_result if isinstance(listener_result, int) else 503 response = Response() response.status_code = code response._content = "" response.headers["Content-Length"] = "0" append_cors_headers(request_headers=headers, response=response) return response # perform the actual invocation of the backend service headers_to_send = None data_to_send = None method_to_send = None if response is None: headers_to_send = handler_chain_request.headers headers_to_send["Connection"] = headers_to_send.get( "Connection") or "close" data_to_send = handler_chain_request.data method_to_send = handler_chain_request.method request_url = get_proxy_backend_url(handler_chain_request.path, run_listeners=True) if modified_request_to_backend: if modified_request_to_backend.url: request_url = get_proxy_backend_url( modified_request_to_backend.url, original_url=request_url) data_to_send = modified_request_to_backend.data if modified_request_to_backend.method: method_to_send = modified_request_to_backend.method # make sure we drop "chunked" transfer encoding from the headers to be forwarded headers_to_send.pop("Transfer-Encoding", None) response = requests.request( method_to_send, url=request_url, data=data_to_send, headers=headers_to_send, stream=True, verify=False, ) # prevent requests from processing response body (e.g., to pass-through gzip encoded content # unmodified) not_consumed = not getattr(response, "_content_consumed", True) pass_raw = not_consumed or response.headers.get("content-encoding") in [ "gzip" ] if pass_raw and getattr(response, "raw", None): new_content = response.raw.read() if new_content: response._content = new_content # run outbound handlers (post-invocation) for listener in listeners_outbound: updated_response = listener.return_response( method=method_to_send or handler_chain_request.method, path=handler_chain_request.path, data=data_to_send or handler_chain_request.data, headers=headers_to_send or handler_chain_request.headers, response=response, ) message_modifier = isinstance(listener, MessageModifyingProxyListener) if message_modifier and isinstance(updated_response, RoutingResponse): # update the fields from updated_response in final response response.status_code = updated_response.status_code or response.status_code response.headers = updated_response.headers or response.headers if isinstance(updated_response.content, (str, bytes)): response._content = updated_response.content if isinstance(updated_response, Response): response = updated_response # allow pre-flight CORS headers by default from localstack.services.s3.s3_listener import ProxyListenerS3 is_s3_listener = any( isinstance(service_listener, ProxyListenerS3) for service_listener in listeners) if not is_s3_listener: append_cors_headers(request_headers=headers, response=response) return response
def modify_and_forward( method=None, path=None, data_bytes=None, headers=None, forward_base_url=None, listeners=None, request_handler=None, client_address=None, server_address=None, ): """This is the central function that coordinates the incoming/outgoing messages with the proxy listeners (message interceptors).""" # Check origin / referer header before anything else happens. if (not config.DISABLE_CORS_CHECKS and should_enforce_self_managed_service( method, path, headers, data_bytes) and not is_cors_origin_allowed(headers)): LOG.info( "Blocked cors request from forbidden origin %s", headers.get("origin") or headers.get("referer"), ) return cors_error_response() listeners = ProxyListener.DEFAULT_LISTENERS + (listeners or []) listeners = [lis for lis in listeners if lis] data = data_bytes def is_full_url(url): return re.match(r"[a-zA-Z]+://.+", url) if is_full_url(path): path = path.split("://", 1)[1] path = "/%s" % (path.split("/", 1)[1] if "/" in path else "") proxy_url = "%s%s" % (forward_base_url, path) for listener in listeners: proxy_url = listener.get_forward_url(method, path, data, headers) or proxy_url target_url = path if not is_full_url(target_url): target_url = "%s%s" % (forward_base_url, target_url) # update original "Host" header (moto s3 relies on this behavior) if not headers.get("Host"): headers["host"] = urlparse(target_url).netloc headers["X-Forwarded-For"] = build_x_forwarded_for(headers, client_address, server_address) response = None modified_request = None # update listener (pre-invocation) for listener in listeners: try: listener_result = listener.forward_request(method=method, path=path, data=data, headers=headers) except HTTPException as e: # TODO: implement properly using exception handlers return http_exception_to_response(e) if isinstance(listener_result, Response): response = listener_result break if isinstance(listener_result, LambdaResponse): response = listener_result break if isinstance(listener_result, dict): response = Response() response._content = json.dumps(json_safe(listener_result)) response.headers["Content-Type"] = APPLICATION_JSON response.status_code = 200 break elif isinstance(listener_result, Request): modified_request = listener_result data = modified_request.data headers = modified_request.headers break elif http2_server.get_async_generator_result(listener_result): return listener_result elif listener_result is not True: # get status code from response, or use Bad Gateway status code code = listener_result if isinstance(listener_result, int) else 503 response = Response() response.status_code = code response._content = "" response.headers["Content-Length"] = "0" append_cors_headers(request_headers=headers, response=response) return response # perform the actual invocation of the backend service if response is None: headers["Connection"] = headers.get("Connection") or "close" data_to_send = data_bytes request_url = proxy_url if modified_request: if modified_request.url: request_url = "%s%s" % (forward_base_url, modified_request.url) data_to_send = modified_request.data # make sure we drop "chunked" transfer encoding from the headers to be forwarded headers.pop("Transfer-Encoding", None) response = requests.request(method, request_url, data=data_to_send, headers=headers, stream=True, verify=False) # prevent requests from processing response body (e.g., to pass-through gzip encoded content unmodified) pass_raw = (hasattr(response, "_content_consumed") and not response._content_consumed ) or response.headers.get("content-encoding") in ["gzip"] if pass_raw and getattr(response, "raw", None): new_content = response.raw.read() if new_content: response._content = new_content # update listener (post-invocation) if listeners: update_listener = listeners[-1] kwargs = { "method": method, "path": path, "data": data_bytes, "headers": headers, "response": response, } if "request_handler" in inspect.getfullargspec( update_listener.return_response).args: # some listeners (e.g., sqs_listener.py) require additional details like the original # request port, hence we pass in a reference to this request handler as well. kwargs["request_handler"] = request_handler updated_response = update_listener.return_response(**kwargs) if isinstance(updated_response, Response): response = updated_response # allow pre-flight CORS headers by default from localstack.services.s3.s3_listener import ProxyListenerS3 is_s3_listener = any([ isinstance(service_listener, ProxyListenerS3) for service_listener in listeners ]) if not is_s3_listener: append_cors_headers(request_headers=headers, response=response) return response