def start(self): with self._lifecycle_lock: if self._started: return self._started = True start_thread(self._run_elasticsearch)
def _run_proxy_and_command(cmd, port, backend_port, update_listener, asynchronous): global PROCESS_THREAD log_startup_message("Kinesis") start_proxy_for_service("kinesis", port, backend_port, update_listener) # TODO: generalize into service manager once it is introduced try: PROCESS_THREAD = do_run(cmd, asynchronous) finally: if asynchronous: def _return_listener(*_): try: ret_code = PROCESS_THREAD.result_future.result() if ret_code not in [0, None]: LOGGER.error("kinesis terminated with return code %s", ret_code) finally: kinesis_stopped.set() start_thread(_return_listener) else: kinesis_stopped.set() return PROCESS_THREAD
def publish_message(topic_arn, req_data, headers, subscription_arn=None, skip_checks=False): sns_backend = SNSBackend.get() message = req_data["Message"][0] message_id = str(uuid.uuid4()) if topic_arn and ":endpoint/" in topic_arn: # cache messages published to platform endpoints cache = sns_backend.platform_endpoint_messages[topic_arn] = ( sns_backend.platform_endpoint_messages.get(topic_arn) or []) cache.append(req_data) LOG.debug("Publishing message to TopicArn: %s | Message: %s", topic_arn, message) start_thread(lambda _: message_to_subscribers( message_id, message, topic_arn, req_data, headers, subscription_arn, skip_checks, )) return message_id
def publish_message(topic_arn, req_data, subscription_arn=None, skip_checks=False): message = req_data['Message'][0] message_id = str(uuid.uuid4()) if topic_arn and ':endpoint/' in topic_arn: # cache messages published to platform endpoints cache = PLATFORM_ENDPOINT_MESSAGES[topic_arn] = PLATFORM_ENDPOINT_MESSAGES.get(topic_arn) or [] cache.append(req_data) LOG.debug('Publishing message to TopicArn: %s | Message: %s' % (topic_arn, message)) start_thread( lambda _: message_to_subscribers(message_id, message, topic_arn, req_data, subscription_arn, skip_checks)) return message_id
def run_process_as_sudo(component, port, asynchronous=False): # make sure we can run sudo commands try: ensure_can_use_sudo() except Exception as e: LOG.error("cannot start service on privileged port %s: %s", port, str(e)) return # start the process as sudo sudo_cmd = "sudo -n " python_cmd = sys.executable edge_url = config.get_edge_url() cmd = "%sPYTHONPATH=.:%s EDGE_FORWARD_URL=%s EDGE_BIND_HOST=%s %s %s %s %s" % ( sudo_cmd, LOCALSTACK_ROOT_FOLDER, edge_url, config.EDGE_BIND_HOST, python_cmd, __file__, component, port, ) def run_command(*_): run(cmd, outfile=subprocess.PIPE, print_error=False) result = start_thread(run_command, quiet=True) if asynchronous else run_command() return result
def run_process_as_sudo(component, port, asynchronous=False, env_vars=None): # make sure we can run sudo commands try: ensure_can_use_sudo() except Exception as e: LOG.error("cannot start service on privileged port %s: %s", port, str(e)) return # prepare environment env_vars = env_vars or {} env_vars["PYTHONPATH"] = f".:{LOCALSTACK_ROOT_FOLDER}" env_vars["EDGE_FORWARD_URL"] = config.get_edge_url() env_vars["EDGE_BIND_HOST"] = config.EDGE_BIND_HOST env_vars_str = env_vars_to_string(env_vars) # start the process as sudo sudo_cmd = "sudo -n" python_cmd = sys.executable cmd = [ sudo_cmd, env_vars_str, python_cmd, __file__, component, str(port), ] shell_cmd = " ".join(cmd) def run_command(*_): run(shell_cmd, outfile=subprocess.PIPE, print_error=False, env_vars=env_vars) LOG.debug("Running command as sudo: %s", shell_cmd) result = start_thread(run_command, quiet=True) if asynchronous else run_command() return result
def serve_flask_app(app, port, host=None, cors=True, asynchronous=False): if cors: CORS(app) if not config.DEBUG: logging.getLogger("werkzeug").setLevel(logging.ERROR) if not host: host = "0.0.0.0" ssl_context = None if not config.FORWARD_EDGE_INMEM: ssl_context = GenericProxy.get_flask_ssl_context(serial_number=port) app.config["ENV"] = "development" def noecho(*args, **kwargs): pass try: import click click.echo = noecho except Exception: pass def _run(*_): app.run(port=int(port), threaded=True, host=host, ssl_context=ssl_context) return app if asynchronous: return start_thread(_run) return _run()
def test_cleanup_threads_and_processes_calls_shutdown_hooks(self): # TODO: move all run/concurrency related tests into separate class started = threading.Event() done = threading.Event() def run_method(*args, **kwargs): started.set() func_thread = kwargs["_thread"] # thread waits until it is stopped func_thread._stop_event.wait() done.set() common.start_thread(run_method) assert started.wait(timeout=2) common.cleanup_threads_and_processes() assert done.wait(timeout=2)
def start_local_api(name, port, method, asynchronous=False): print( 'Starting mock %s service on %s ports %s (recommended) and %s (deprecated)...' % (name, get_service_protocol(), config.EDGE_PORT, port)) if asynchronous: thread = start_thread(method, port, quiet=True) return thread else: method(port)
def _create_cluster(self, arn, url, version, create_domain_request) -> Server: with self.mutex: if not self.cluster: # startup routine for the singleton cluster instance self.cluster = ElasticsearchCluster( port=get_free_tcp_port(), directories=resolve_directories(version, arn)) def _start_async(*_): LOG.info("starting %s on %s", type(self.cluster), self.cluster.url) self.cluster.start() # start may block during install start_thread(_start_async) return ClusterEndpoint(self.cluster, EndpointProxy(url, self.cluster.url))
def start(self): if self.thread: return def _run(*_args): self.running = True self.run() self.thread = start_thread(_run)
def start_local_api(name, port, api, method, asynchronous=False): print('Starting mock %s service on %s ...' % (name, edge_ports_info())) if config.FORWARD_EDGE_INMEM: port = get_free_tcp_port() PROXY_LISTENERS[api] = (api, port, None) if asynchronous: thread = start_thread(method, port, quiet=True) return thread else: method(port)
def start_local_api(name, port, api, method, asynchronous=False): log_startup_message(name) if config.FORWARD_EDGE_INMEM: port = get_free_tcp_port() PROXY_LISTENERS[api] = (api, port, None) if asynchronous: thread = start_thread(method, port, quiet=True) return thread else: method(port)
def _create_cluster(self, arn, url, version) -> Server: with self.mutex: if not self.cluster: engine_type = versions.get_engine_type(version) # startup routine for the singleton cluster instance if engine_type == EngineType.OpenSearch: self.cluster = OpensearchCluster( port=get_free_tcp_port(), directories=resolve_directories(version, arn) ) else: self.cluster = ElasticsearchCluster( port=get_free_tcp_port(), directories=resolve_directories(version, arn) ) def _start_async(*_): LOG.info("starting %s on %s", type(self.cluster), self.cluster.url) self.cluster.start() # start may block during install start_thread(_start_async) cluster_endpoint = ClusterEndpoint(self.cluster, EndpointProxy(url, self.cluster.url)) self.clusters[arn] = cluster_endpoint return cluster_endpoint
def do_start_thread(self) -> FuncThread: """ Creates and starts the thread running the server. By default, it calls the do_run method in a FuncThread, but can be overridden to if the subclass wants to return its own thread. """ def _run(*_): try: return self.do_run() except StopServer: LOG.debug("stopping server %s", self.url) finally: self._stopped.set() return start_thread(_run)
def _do_start_retry(self, *_): # TODO: actually retry try: if config.DEBUG_ANALYTICS: LOG.debug("trying to register session with analytics backend") response = self._client.start_session(get_client_metadata()) if config.DEBUG_ANALYTICS: LOG.debug("session endpoint returned: %s", response) except Exception: self.tracking_disabled = True if config.DEBUG_ANALYTICS: LOG.exception( "error while registering session. disabling tracking") return finally: self._startup_complete = True start_thread(self.run) def _do_close(): self.close_sync(timeout=2) atexit.register(_do_close)
def start_runtime_components(): from localstack.services.edge import start_edge from localstack.services.internal import LocalstackResourceHandler, get_internal_apis # serve internal APIs through the generic proxy ProxyListener.DEFAULT_LISTENERS.append(LocalstackResourceHandler(get_internal_apis())) # TODO: we want a composable LocalStack runtime (edge proxy, service manager, dns, ...) t = start_thread(start_edge, quiet=False) # TODO: properly encapsulate starting/stopping of edge server in a class if not poll_condition( lambda: is_port_open(config.get_edge_port_http()), timeout=5, interval=0.1 ): raise TimeoutError( f"gave up waiting for edge server on {config.EDGE_BIND_HOST}:{config.EDGE_PORT}" ) return t
def run(self, *_): flush_scheduler = start_thread(self._run_flush_schedule) try: while True: command = self._command_queue.get() if command is self._cmd_flush or command is self._cmd_stop: try: self._do_flush() except Exception: if config.DEBUG_ANALYTICS: LOG.exception("error while flushing events") if command is self._cmd_stop: return finally: self._stopped.set() flush_scheduler.stop()