def start_proxy(port, backend_url, update_listener=None, quiet=False, params={}, use_ssl=None):
use_ssl = config.USE_SSL if use_ssl is None else use_ssl
proxy_thread = GenericProxy(port=port, forward_url=backend_url,
ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def start_proxy(port, backend_url, update_listener, quiet=False, params={}):
proxy_thread = GenericProxy(port=port,
forward_url=backend_url,
ssl=USE_SSL,
update_listener=update_listener,
quiet=quiet,
params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def start_proxy(port,
backend_port,
update_listener,
quiet=False,
backend_host=DEFAULT_BACKEND_HOST,
params={}):
proxy_thread = GenericProxy(port=port,
forward_host='%s:%s' %
(backend_host, backend_port),
ssl=USE_SSL,
update_listener=update_listener,
quiet=quiet,
params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
def start_ssl_proxy(port, target, target_ssl=False):
import pproxy
from localstack.services.generic_proxy import GenericProxy
if ':' not in str(target):
target = '127.0.0.1:%s' % target
print('Starting SSL proxy server %s -> %s' % (port, target))
# create server and remote connection
server = pproxy.Server('secure+tunnel://0.0.0.0:%s' % port)
target_proto = 'secure+tunnel' if target_ssl else 'tunnel'
remote = pproxy.Connection('%s://%s' % (target_proto, target))
args = dict(rserver=[remote], verbose=print)
# set SSL contexts
_, cert_file_name, key_file_name = GenericProxy.create_ssl_cert()
for context in pproxy.server.sslcontexts:
context.load_cert_chain(cert_file_name, key_file_name)
loop = asyncio.get_event_loop()
handler = loop.run_until_complete(server.start_server(args))
try:
loop.run_forever()
except KeyboardInterrupt:
print('exit!')
handler.close()
loop.run_until_complete(handler.wait_closed())
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def _do_start_ssl_proxy(port: int, target: PortOrUrl, target_ssl=False):
import pproxy
from localstack.services.generic_proxy import GenericProxy
if ":" not in str(target):
target = "127.0.0.1:%s" % target
LOG.debug("Starting SSL proxy server %s -> %s", port, target)
# create server and remote connection
server = pproxy.Server("secure+tunnel://0.0.0.0:%s" % port)
target_proto = "ssl+tunnel" if target_ssl else "tunnel"
remote = pproxy.Connection("%s://%s" % (target_proto, target))
args = dict(rserver=[remote], verbose=print)
# set SSL contexts
_, cert_file_name, key_file_name = GenericProxy.create_ssl_cert()
for context in pproxy.server.sslcontexts:
context.load_cert_chain(cert_file_name, key_file_name)
loop = ensure_event_loop()
handler = loop.run_until_complete(server.start_server(args))
try:
loop.run_forever()
except KeyboardInterrupt:
print("exit!")
handler.close()
loop.run_until_complete(handler.wait_closed())
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def main():
"""
Serve the LocalstackGateway with the default configuration directly through hypercorn. This is mostly for
development purposes and documentation on how to serve the Gateway.
"""
from .serving.hypercorn import serve
use_ssl = True
port = 4566
# serve the LocalStackAwsGateway in a dev app
from localstack.utils.bootstrap import setup_logging
setup_logging()
if use_ssl:
from localstack.services.generic_proxy import (
GenericProxy,
install_predefined_cert_if_available,
)
install_predefined_cert_if_available()
_, cert_file_name, key_file_name = GenericProxy.create_ssl_cert(
serial_number=port)
ssl_creds = (cert_file_name, key_file_name)
else:
ssl_creds = None
gw = LocalstackAwsGateway(SERVICE_PLUGINS)
serve(gw, use_reloader=True, port=port, ssl_creds=ssl_creds)
def start_http_backend(test_port):
# test listener for target HTTP backend
class TestListener(ProxyListener):
def forward_request(self, **kwargs):
response = Response()
response.status_code = 200
result = {
'data': kwargs.get('data') or '{}',
'headers': dict(kwargs.get('headers'))
}
response._content = json.dumps(json_safe(result))
return response
proxy = GenericProxy(test_port, update_listener=TestListener())
proxy.start()
return proxy
def serve(port, quiet=True):
if quiet:
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
ssl_context = GenericProxy.get_flask_ssl_context()
app.run(port=int(port),
threaded=True,
host='0.0.0.0',
ssl_context=ssl_context)
def run_parallel_download():
file_length = 10000000
class DownloadListener(ProxyListener):
def forward_request(self, method, path, data, headers):
sleep_time = int(path.replace('/', ''))
time.sleep(sleep_time)
response = Response()
response.status_code = 200
response._content = ('%s' % sleep_time) * file_length
return response
test_port = 12124
tmp_file_pattern = '/tmp/test.%s'
proxy = GenericProxy(port=test_port, update_listener=DownloadListener())
proxy.start()
def do_download(param):
tmp_file = tmp_file_pattern % param
TMP_FILES.append(tmp_file)
download('http://localhost:%s/%s' % (test_port, param), tmp_file)
values = (1, 2, 3)
parallelize(do_download, values)
proxy.stop()
for val in values:
tmp_file = tmp_file_pattern % val
assert len(load_file(tmp_file)) == file_length
def test_api_gateway_http_integration():
test_port = 12123
backend_url = 'http://localhost:%s%s' % (test_port, API_PATH_HTTP_BACKEND)
# create target HTTP backend
class TestListener(ProxyListener):
def forward_request(self, **kwargs):
response = Response()
response.status_code = 200
response._content = kwargs.get('data') or '{}'
return response
proxy = GenericProxy(test_port, update_listener=TestListener())
proxy.start()
# create API Gateway and connect it to the HTTP backend
result = connect_api_gateway_to_http('test_gateway2', backend_url, path=API_PATH_HTTP_BACKEND)
url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'],
stage_name=TEST_STAGE_NAME, path=API_PATH_HTTP_BACKEND)
# make sure CORS headers are present
origin = 'localhost'
result = requests.options(url, headers={'origin': origin})
assert result.status_code == 200
assert re.match(result.headers['Access-Control-Allow-Origin'].replace('*', '.*'), origin)
assert 'POST' in result.headers['Access-Control-Allow-Methods']
# make test request to gateway
result = requests.get(url)
assert result.status_code == 200
assert to_str(result.content) == '{}'
data = {'data': 123}
result = requests.post(url, data=json.dumps(data))
assert result.status_code == 200
assert json.loads(to_str(result.content)) == data
# clean up
proxy.stop()
def start_server(port, asynchronous=False):
if is_port_open(port):
LOG.debug('API Multiserver appears to be already running.')
return
class ConfigListener(ProxyListener):
def forward_request(self, method, path, data, **kwargs):
response = Response()
response.status_code = 200
response._content = '{}'
try:
if path == API_PATH_SERVERS:
if method == 'POST':
start_api_server_locally(json.loads(to_str(data)))
elif method == 'GET':
response._content = json.dumps(json_safe(API_SERVERS))
except Exception as e:
LOG.error('Unable to process request: %s' % e)
response.status_code = 500
response._content = str(e)
return response
proxy = GenericProxy(port, update_listener=ConfigListener())
proxy.start()
if asynchronous:
return proxy
proxy.join()
def test_api_gateway_http_integration():
test_port = 12123
backend_url = 'http://localhost:%s%s' % (test_port, API_PATH_HTTP_BACKEND)
# create target HTTP backend
def listener(**kwargs):
response = Response()
response.status_code = 200
response._content = json.dumps(
kwargs['data']) if kwargs['data'] else '{}'
return response
proxy = GenericProxy(test_port, update_listener=listener)
proxy.start()
# create API Gateway and connect it to the HTTP backend
result = connect_api_gateway_to_http('test_gateway2',
backend_url,
path=API_PATH_HTTP_BACKEND)
# make test request to gateway
url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'],
stage_name=TEST_STAGE_NAME,
path=API_PATH_HTTP_BACKEND)
result = requests.get(url)
assert result.status_code == 200
assert to_str(result.content) == '{}'
data = {"data": 123}
result = requests.post(url, data=json.dumps(data))
assert result.status_code == 200
assert json.loads(to_str(result.content)) == data
# clean up
proxy.stop()
def _do_start_ssl_proxy(
port: int,
target: PortOrUrl,
target_ssl=False,
client_cert_key: Tuple[str, str] = None,
bind_address: str = "0.0.0.0",
):
"""
Starts a tcp proxy (with tls) on the specified port
:param port: Port the proxy should bind to
:param target: Target of the proxy. If a port, it will connect to localhost:
:param target_ssl: Specify if the proxy should connect to the target using SSL/TLS
:param client_cert_key: Client certificate for the target connection. Only set if target_ssl=True
:param bind_address: Bind address of the proxy server
"""
import pproxy
from localstack.services.generic_proxy import GenericProxy
if ":" not in str(target):
target = f"127.0.0.1:{target}"
LOG.debug("Starting SSL proxy server %s -> %s", port, target)
# create server and remote connection
server = pproxy.Server(f"secure+tunnel://{bind_address}:{port}")
target_proto = "ssl+tunnel" if target_ssl else "tunnel"
remote = pproxy.Connection(f"{target_proto}://{target}")
if client_cert_key:
# TODO verify client certs server side?
LOG.debug("Configuring ssl proxy to use client certs")
cert_file, key_file = _save_cert_keys(client_cert_key=client_cert_key)
remote.sslclient.load_cert_chain(certfile=cert_file, keyfile=key_file)
args = dict(rserver=[remote])
# set SSL contexts
_, cert_file_name, key_file_name = GenericProxy.create_ssl_cert()
for context in pproxy.server.sslcontexts:
context.load_cert_chain(cert_file_name, key_file_name)
loop = ensure_event_loop()
handler = loop.run_until_complete(server.start_server(args))
try:
loop.run_forever()
except KeyboardInterrupt:
print("exit!")
handler.close()
loop.run_until_complete(handler.wait_closed())
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def do_start_edge(port, use_ssl, asynchronous=False):
# get port and start Edge
print('Starting edge router (http%s port %s)...' % ('s' if use_ssl else '', port))
# use use=True here because our proxy allows both, HTTP and HTTPS traffic
proxy = GenericProxy(port, ssl=True, update_listener=ProxyListenerEdge())
proxy.start()
if not asynchronous:
proxy.join()
return proxy
def test_api_gateway_http_integration(self):
test_port = 12123
backend_url = 'http://localhost:%s%s' % (test_port, self.API_PATH_HTTP_BACKEND)
# create target HTTP backend
class TestListener(ProxyListener):
def forward_request(self, **kwargs):
response = Response()
response.status_code = 200
response._content = kwargs.get('data') or '{}'
return response
proxy = GenericProxy(test_port, update_listener=TestListener())
proxy.start()
# create API Gateway and connect it to the HTTP backend
result = self.connect_api_gateway_to_http(
'test_gateway2',
backend_url,
path=self.API_PATH_HTTP_BACKEND
)
url = INBOUND_GATEWAY_URL_PATTERN.format(
api_id=result['id'],
stage_name=self.TEST_STAGE_NAME,
path=self.API_PATH_HTTP_BACKEND
)
# make sure CORS headers are present
origin = 'localhost'
result = requests.options(url, headers={'origin': origin})
self.assertEqual(result.status_code, 200)
self.assertTrue(re.match(result.headers['Access-Control-Allow-Origin'].replace('*', '.*'), origin))
self.assertIn('POST', result.headers['Access-Control-Allow-Methods'])
# make test request to gateway
result = requests.get(url)
self.assertEqual(result.status_code, 200)
self.assertEqual(to_str(result.content), '{}')
data = {'data': 123}
result = requests.post(url, data=json.dumps(data))
self.assertEqual(result.status_code, 200)
self.assertEqual(json.loads(to_str(result.content)), data)
# clean up
proxy.stop()
def do_start_edge(port, use_ssl, asynchronous=False):
try:
# start local DNS server, if present
from localstack_ext.services import dns_server
dns_server.start_servers()
except Exception:
pass
# get port and start Edge
print('Starting edge router (http%s port %s)...' % ('s' if use_ssl else '', port))
# use use=True here because our proxy allows both, HTTP and HTTPS traffic
proxy = GenericProxy(port, ssl=True, update_listener=ProxyListenerEdge())
proxy.start()
if not asynchronous:
proxy.join()
return proxy
def serve_gateway(bind_address, port, use_ssl, asynchronous=False):
"""
Implementation of the edge.do_start_edge_proxy interface to start a Hypercorn server instance serving the
LocalstackAwsGateway.
"""
from hypercorn import Config
from localstack.aws.app import LocalstackAwsGateway
from localstack.aws.serving.asgi import AsgiGateway
from localstack.http.hypercorn import HypercornServer
from localstack.services.generic_proxy import GenericProxy, install_predefined_cert_if_available
# build server config
config = Config()
if isinstance(bind_address, str):
bind_address = [bind_address]
config.bind = [f"{addr}:{port}" for addr in bind_address]
if use_ssl:
install_predefined_cert_if_available()
_, cert_file_name, key_file_name = GenericProxy.create_ssl_cert(serial_number=port)
config.certfile = cert_file_name
config.keyfile = key_file_name
# build gateway
loop = asyncio.new_event_loop()
app = AsgiGateway(LocalstackAwsGateway(SERVICE_PLUGINS), event_loop=loop)
# start serving gateway
server = HypercornServer(app, config, loop)
server.start()
if not asynchronous:
server.join()
return server._thread
def start_server(port):
class ConfigListener(ProxyListener):
def forward_request(self, method, path, data, **kwargs):
response = Response()
response.status_code = 200
response._content = '{}'
try:
if path == API_PATH_SERVERS:
if method == 'POST':
start_api_server_locally(json.loads(to_str(data)))
elif method == 'GET':
response._content = json.dumps(json_safe(API_SERVERS))
except Exception as e:
LOG.error('Unable to process request: %s' % e)
response.status_code = 500
response._content = str(e)
return response
proxy = GenericProxy(port, update_listener=ConfigListener())
proxy.start()
proxy.join()
def start_proxy(port, backend_url, update_listener, quiet=False, params={}):
proxy_thread = GenericProxy(port=port, forward_url=backend_url,
ssl=USE_SSL, update_listener=update_listener, quiet=quiet, params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def test_api_gateway_http_integration(self):
test_port = 12123
backend_url = 'http://localhost:%s%s' % (test_port,
self.API_PATH_HTTP_BACKEND)
# create target HTTP backend
class TestListener(ProxyListener):
def forward_request(self, **kwargs):
response = Response()
response.status_code = 200
result = {
'data': kwargs.get('data') or '{}',
'headers': dict(kwargs.get('headers'))
}
response._content = json.dumps(json_safe(result))
return response
proxy = GenericProxy(test_port, update_listener=TestListener())
proxy.start()
# create API Gateway and connect it to the HTTP backend
result = self.connect_api_gateway_to_http(
'test_gateway2', backend_url, path=self.API_PATH_HTTP_BACKEND)
url = self.gateway_request_url(api_id=result['id'],
stage_name=self.TEST_STAGE_NAME,
path=self.API_PATH_HTTP_BACKEND)
# make sure CORS headers are present
origin = 'localhost'
result = requests.options(url, headers={'origin': origin})
self.assertEqual(result.status_code, 200)
self.assertTrue(
re.match(
result.headers['Access-Control-Allow-Origin'].replace(
'*', '.*'), origin))
self.assertIn('POST', result.headers['Access-Control-Allow-Methods'])
# make test GET request to gateway
result = requests.get(url)
self.assertEqual(result.status_code, 200)
self.assertEqual(json.loads(to_str(result.content))['data'], '{}')
# make test POST request to gateway
data = json.dumps({'data': 123})
result = requests.post(url, data=data)
self.assertEqual(result.status_code, 200)
self.assertEqual(json.loads(to_str(result.content))['data'], data)
# make test POST request with non-JSON content type
data = 'test=123'
ctype = 'application/x-www-form-urlencoded'
result = requests.post(url, data=data, headers={'content-type': ctype})
self.assertEqual(result.status_code, 200)
content = json.loads(to_str(result.content))
headers = CaseInsensitiveDict(content['headers'])
self.assertEqual(content['data'], data)
self.assertEqual(headers['content-type'], ctype)
# clean up
proxy.stop()
install.install_elasticsearch()
backend_port = DEFAULT_PORT_ELASTICSEARCH_BACKEND
es_data_dir = '%s/infra/elasticsearch/data' % (ROOT_PATH)
if DATA_DIR:
es_data_dir = '%s/elasticsearch' % DATA_DIR
# Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments,
# hence we use the default bind address 127.0.0.0 and put a proxy in front of it
cmd = ((
'ES_JAVA_OPTS=\"$ES_JAVA_OPTS -Xms200m -Xmx500m\" %s/infra/elasticsearch/bin/elasticsearch '
+
'-E http.port=%s -E http.publish_port=%s -E http.compression=false -E path.data=%s '
+ '-E xpack.security.enabled=false') %
(ROOT_PATH, backend_port, backend_port, es_data_dir))
if USE_SSL:
# make sure we have a test cert generated and configured
GenericProxy.create_ssl_cert()
cmd += ((' -E xpack.ssl.key=%s.key -E xpack.ssl.certificate=%s.crt ' +
'-E xpack.security.transport.ssl.enabled=true ' +
'-E xpack.security.http.ssl.enabled=true') %
(SERVER_CERT_PEM_FILE, SERVER_CERT_PEM_FILE))
print("Starting local Elasticsearch (%s port %s)..." %
(get_service_protocol(), port))
if delete_data:
run('rm -rf %s' % es_data_dir)
# fix permissions
run('chmod -R 777 %s/infra/elasticsearch' % ROOT_PATH)
run('mkdir -p "%s"; chmod -R 777 "%s"' % (es_data_dir, es_data_dir))
# start proxy and ES process
start_proxy(port,
backend_port,
update_listener,
