def start(self):
with self._lifecycle_lock:
if self._started:
return
self._started = True
start_thread(self._run_elasticsearch)
def _run_proxy_and_command(cmd, port, backend_port, update_listener,
asynchronous):
global PROCESS_THREAD
log_startup_message("Kinesis")
start_proxy_for_service("kinesis", port, backend_port, update_listener)
# TODO: generalize into service manager once it is introduced
try:
PROCESS_THREAD = do_run(cmd, asynchronous)
finally:
if asynchronous:
def _return_listener(*_):
try:
ret_code = PROCESS_THREAD.result_future.result()
if ret_code not in [0, None]:
LOGGER.error("kinesis terminated with return code %s",
ret_code)
finally:
kinesis_stopped.set()
start_thread(_return_listener)
else:
kinesis_stopped.set()
return PROCESS_THREAD
def publish_message(topic_arn,
req_data,
headers,
subscription_arn=None,
skip_checks=False):
sns_backend = SNSBackend.get()
message = req_data["Message"][0]
message_id = str(uuid.uuid4())
if topic_arn and ":endpoint/" in topic_arn:
# cache messages published to platform endpoints
cache = sns_backend.platform_endpoint_messages[topic_arn] = (
sns_backend.platform_endpoint_messages.get(topic_arn) or [])
cache.append(req_data)
LOG.debug("Publishing message to TopicArn: %s | Message: %s", topic_arn,
message)
start_thread(lambda _: message_to_subscribers(
message_id,
message,
topic_arn,
req_data,
headers,
subscription_arn,
skip_checks,
))
return message_id
def publish_message(topic_arn, req_data, subscription_arn=None, skip_checks=False):
message = req_data['Message'][0]
message_id = str(uuid.uuid4())
if topic_arn and ':endpoint/' in topic_arn:
# cache messages published to platform endpoints
cache = PLATFORM_ENDPOINT_MESSAGES[topic_arn] = PLATFORM_ENDPOINT_MESSAGES.get(topic_arn) or []
cache.append(req_data)
LOG.debug('Publishing message to TopicArn: %s | Message: %s' % (topic_arn, message))
start_thread(
lambda _: message_to_subscribers(message_id, message, topic_arn, req_data, subscription_arn, skip_checks))
return message_id
def run_process_as_sudo(component, port, asynchronous=False):
# make sure we can run sudo commands
try:
ensure_can_use_sudo()
except Exception as e:
LOG.error("cannot start service on privileged port %s: %s", port,
str(e))
return
# start the process as sudo
sudo_cmd = "sudo -n "
python_cmd = sys.executable
edge_url = config.get_edge_url()
cmd = "%sPYTHONPATH=.:%s EDGE_FORWARD_URL=%s EDGE_BIND_HOST=%s %s %s %s %s" % (
sudo_cmd,
LOCALSTACK_ROOT_FOLDER,
edge_url,
config.EDGE_BIND_HOST,
python_cmd,
__file__,
component,
port,
)
def run_command(*_):
run(cmd, outfile=subprocess.PIPE, print_error=False)
result = start_thread(run_command,
quiet=True) if asynchronous else run_command()
return result
def run_process_as_sudo(component, port, asynchronous=False, env_vars=None):
# make sure we can run sudo commands
try:
ensure_can_use_sudo()
except Exception as e:
LOG.error("cannot start service on privileged port %s: %s", port, str(e))
return
# prepare environment
env_vars = env_vars or {}
env_vars["PYTHONPATH"] = f".:{LOCALSTACK_ROOT_FOLDER}"
env_vars["EDGE_FORWARD_URL"] = config.get_edge_url()
env_vars["EDGE_BIND_HOST"] = config.EDGE_BIND_HOST
env_vars_str = env_vars_to_string(env_vars)
# start the process as sudo
sudo_cmd = "sudo -n"
python_cmd = sys.executable
cmd = [
sudo_cmd,
env_vars_str,
python_cmd,
__file__,
component,
str(port),
]
shell_cmd = " ".join(cmd)
def run_command(*_):
run(shell_cmd, outfile=subprocess.PIPE, print_error=False, env_vars=env_vars)
LOG.debug("Running command as sudo: %s", shell_cmd)
result = start_thread(run_command, quiet=True) if asynchronous else run_command()
return result
def serve_flask_app(app, port, host=None, cors=True, asynchronous=False):
if cors:
CORS(app)
if not config.DEBUG:
logging.getLogger("werkzeug").setLevel(logging.ERROR)
if not host:
host = "0.0.0.0"
ssl_context = None
if not config.FORWARD_EDGE_INMEM:
ssl_context = GenericProxy.get_flask_ssl_context(serial_number=port)
app.config["ENV"] = "development"
def noecho(*args, **kwargs):
pass
try:
import click
click.echo = noecho
except Exception:
pass
def _run(*_):
app.run(port=int(port),
threaded=True,
host=host,
ssl_context=ssl_context)
return app
if asynchronous:
return start_thread(_run)
return _run()
def test_cleanup_threads_and_processes_calls_shutdown_hooks(self):
# TODO: move all run/concurrency related tests into separate class
started = threading.Event()
done = threading.Event()
def run_method(*args, **kwargs):
started.set()
func_thread = kwargs["_thread"]
# thread waits until it is stopped
func_thread._stop_event.wait()
done.set()
common.start_thread(run_method)
assert started.wait(timeout=2)
common.cleanup_threads_and_processes()
assert done.wait(timeout=2)
def start_local_api(name, port, method, asynchronous=False):
print(
'Starting mock %s service on %s ports %s (recommended) and %s (deprecated)...'
% (name, get_service_protocol(), config.EDGE_PORT, port))
if asynchronous:
thread = start_thread(method, port, quiet=True)
return thread
else:
method(port)
def _create_cluster(self, arn, url, version,
create_domain_request) -> Server:
with self.mutex:
if not self.cluster:
# startup routine for the singleton cluster instance
self.cluster = ElasticsearchCluster(
port=get_free_tcp_port(),
directories=resolve_directories(version, arn))
def _start_async(*_):
LOG.info("starting %s on %s", type(self.cluster),
self.cluster.url)
self.cluster.start() # start may block during install
start_thread(_start_async)
return ClusterEndpoint(self.cluster,
EndpointProxy(url, self.cluster.url))
def start(self):
if self.thread:
return
def _run(*_args):
self.running = True
self.run()
self.thread = start_thread(_run)
def start_local_api(name, port, api, method, asynchronous=False):
print('Starting mock %s service on %s ...' % (name, edge_ports_info()))
if config.FORWARD_EDGE_INMEM:
port = get_free_tcp_port()
PROXY_LISTENERS[api] = (api, port, None)
if asynchronous:
thread = start_thread(method, port, quiet=True)
return thread
else:
method(port)
def start_local_api(name, port, api, method, asynchronous=False):
log_startup_message(name)
if config.FORWARD_EDGE_INMEM:
port = get_free_tcp_port()
PROXY_LISTENERS[api] = (api, port, None)
if asynchronous:
thread = start_thread(method, port, quiet=True)
return thread
else:
method(port)
def _create_cluster(self, arn, url, version) -> Server:
with self.mutex:
if not self.cluster:
engine_type = versions.get_engine_type(version)
# startup routine for the singleton cluster instance
if engine_type == EngineType.OpenSearch:
self.cluster = OpensearchCluster(
port=get_free_tcp_port(), directories=resolve_directories(version, arn)
)
else:
self.cluster = ElasticsearchCluster(
port=get_free_tcp_port(), directories=resolve_directories(version, arn)
)
def _start_async(*_):
LOG.info("starting %s on %s", type(self.cluster), self.cluster.url)
self.cluster.start() # start may block during install
start_thread(_start_async)
cluster_endpoint = ClusterEndpoint(self.cluster, EndpointProxy(url, self.cluster.url))
self.clusters[arn] = cluster_endpoint
return cluster_endpoint
def do_start_thread(self) -> FuncThread:
"""
Creates and starts the thread running the server. By default, it calls the do_run method in a FuncThread, but
can be overridden to if the subclass wants to return its own thread.
"""
def _run(*_):
try:
return self.do_run()
except StopServer:
LOG.debug("stopping server %s", self.url)
finally:
self._stopped.set()
return start_thread(_run)
def _do_start_retry(self, *_):
# TODO: actually retry
try:
if config.DEBUG_ANALYTICS:
LOG.debug("trying to register session with analytics backend")
response = self._client.start_session(get_client_metadata())
if config.DEBUG_ANALYTICS:
LOG.debug("session endpoint returned: %s", response)
except Exception:
self.tracking_disabled = True
if config.DEBUG_ANALYTICS:
LOG.exception(
"error while registering session. disabling tracking")
return
finally:
self._startup_complete = True
start_thread(self.run)
def _do_close():
self.close_sync(timeout=2)
atexit.register(_do_close)
def start_runtime_components():
from localstack.services.edge import start_edge
from localstack.services.internal import LocalstackResourceHandler, get_internal_apis
# serve internal APIs through the generic proxy
ProxyListener.DEFAULT_LISTENERS.append(LocalstackResourceHandler(get_internal_apis()))
# TODO: we want a composable LocalStack runtime (edge proxy, service manager, dns, ...)
t = start_thread(start_edge, quiet=False)
# TODO: properly encapsulate starting/stopping of edge server in a class
if not poll_condition(
lambda: is_port_open(config.get_edge_port_http()), timeout=5, interval=0.1
):
raise TimeoutError(
f"gave up waiting for edge server on {config.EDGE_BIND_HOST}:{config.EDGE_PORT}"
)
return t
def run(self, *_):
flush_scheduler = start_thread(self._run_flush_schedule)
try:
while True:
command = self._command_queue.get()
if command is self._cmd_flush or command is self._cmd_stop:
try:
self._do_flush()
except Exception:
if config.DEBUG_ANALYTICS:
LOG.exception("error while flushing events")
if command is self._cmd_stop:
return
finally:
self._stopped.set()
flush_scheduler.stop()
