def start_proxy(port, backend_url, update_listener=None, quiet=False, params={}, use_ssl=None): use_ssl = config.USE_SSL if use_ssl is None else use_ssl proxy_thread = GenericProxy(port=port, forward_url=backend_url, ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params) proxy_thread.start() TMP_THREADS.append(proxy_thread) return proxy_thread
def do_start_thread(self) -> FuncThread: install.install_elasticmq() # create config file config_params = config_template % ( config.LOCALSTACK_HOSTNAME, get_free_tcp_port(), self.port, ) # create temporary config config_file = os.path.join(config.dirs.tmp, "sqs.%s.conf" % short_uid()) LOG.debug("saving config file to %s:\n%s", config_file, config_params) TMP_FILES.append(config_file) save_file(config_file, config_params) # start process cmd = [ "java", f"-Dconfig.file={config_file}", f"-Xmx{self.max_heap_size}", "-jar", os.path.join(self.install_dir, "elasticmq-server.jar"), ] LOG.debug("starting elasticmq server with command %s", " ".join(cmd)) t = ShellCommandThread( cmd, strip_color=True, log_listener=self._log_listener, auto_restart=True, ) TMP_THREADS.append(t) t.start() return t
def start_api_server_locally(request): if localstack_config.FORWARD_EDGE_INMEM: if "__started__" in API_SERVERS: return API_SERVERS["__started__"] = True api = request.get("api") port = request.get("port") if api in API_SERVERS: return API_SERVERS[api] result = API_SERVERS[api] = {} def thread_func(params): if localstack_config.FORWARD_EDGE_INMEM: return moto_server.main( ["-p", str(port), "-H", constants.BIND_HOST]) return moto_server.main( [api, "-p", str(port), "-H", constants.BIND_HOST]) thread = FuncThread(thread_func) thread.start() TMP_THREADS.append(thread) result["port"] = port result["thread"] = thread return result
def start_ssl_proxy( port: int, target: PortOrUrl, target_ssl=False, client_cert_key: Tuple[str, str] = None, asynchronous: bool = False, fix_encoding: bool = False, ): """Start a proxy server that accepts SSL requests and forwards requests to a backend (either SSL or non-SSL)""" if client_cert_key or fix_encoding: # use a custom proxy listener, in case the user provides client certificates for authentication if client_cert_key: server = _do_start_ssl_proxy_with_client_auth( port, target, client_cert_key=client_cert_key) else: server = _do_start_ssl_proxy_with_listener(port, target) if not asynchronous: server.join() return server def _run(*args): return _do_start_ssl_proxy(port, target, target_ssl=target_ssl) if not asynchronous: return _run() proxy = FuncThread(_run) TMP_THREADS.append(proxy) proxy.start() return proxy
def start_edge(port=None, use_ssl=True, asynchronous=False): if not port: port = config.EDGE_PORT if config.EDGE_PORT_HTTP: do_start_edge(config.EDGE_PORT_HTTP, use_ssl=False, asynchronous=True) if port > 1024 or is_root(): return do_start_edge(port, use_ssl, asynchronous=asynchronous) # process requires priviledged port but we're not root -> try running as sudo class Terminator(object): def stop(self, quiet=True): try: url = 'http%s://localhost:%s' % ('s' if use_ssl else '', port) requests.verify_ssl = False requests.post(url, headers={HEADER_KILL_SIGNAL: 'kill'}) except Exception: pass # make sure we can run sudo commands ensure_can_use_sudo() # register a signal handler to terminate the sudo process later on TMP_THREADS.append(Terminator()) # start the process as sudo sudo_cmd = 'sudo ' python_cmd = sys.executable cmd = '%sPYTHONPATH=.:%s %s %s %s' % (sudo_cmd, LOCALSTACK_ROOT_FOLDER, python_cmd, __file__, port) process = run(cmd, asynchronous=asynchronous) return process
def start_edge(port=None, use_ssl=True, asynchronous=False): if not port: port = config.EDGE_PORT if config.EDGE_PORT_HTTP and config.EDGE_PORT_HTTP != port: do_start_edge( config.EDGE_BIND_HOST, config.EDGE_PORT_HTTP, use_ssl=False, asynchronous=True, ) if port > 1024 or is_root(): return do_start_edge(config.EDGE_BIND_HOST, port, use_ssl, asynchronous=asynchronous) # process requires privileged port but we're not root -> try running as sudo class Terminator(object): def stop(self, quiet=True): try: url = "http%s://%s:%s" % ("s" if use_ssl else "", LOCALHOST, port) requests.verify_ssl = False requests.post(url, headers={HEADER_KILL_SIGNAL: "kill"}) except Exception: pass # register a signal handler to terminate the sudo process later on TMP_THREADS.append(Terminator()) return run_process_as_sudo("edge", port, asynchronous=asynchronous)
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={}, endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, region_name=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]): env = aws_stack.get_environment(env) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) # decide which credentials provider to use credentialsProvider = None if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and ('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = 'cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider' # pass through env variables to child process for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if aws_stack.is_local_env(env): # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars['AWS_CBOR_DISABLE'] = 'true' if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace('*', short_uid()) TMP_FILES.append(log_file) run('touch %s' % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level, 'log_prefix': 'KCL', 'log_subscribers': log_subscribers}) reader_thread.start() # construct stream info stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars) props_file = stream_info['properties_file'] # set kcl config options kwargs = { 'metricsLevel': 'NONE', 'initialPositionInStream': 'LATEST' } # set parameters for local connection if aws_stack.is_local_env(env): kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS) kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB) kwargs['kinesisProtocol'] = get_service_protocol() kwargs['dynamodbProtocol'] = get_service_protocol() kwargs['disableCertChecking'] = 'true' kwargs.update(configs) # create config file kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info['app_name'], credentialsProvider=credentialsProvider, region_name=region_name, **kwargs) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer
def start_proxy_server(port, forward_url=None, use_ssl=None, update_listener=None, quiet=False, params={}): if USE_HTTP2_SERVER: return start_proxy_server_http2(port=port, forward_url=forward_url, use_ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params) proxy_thread = GenericProxy(port=port, forward_url=forward_url, ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params) proxy_thread.start() TMP_THREADS.append(proxy_thread) return proxy_thread
def start_local_api(name, port, method, asynchronous=False): print('Starting mock %s service (%s port %s)...' % (name, get_service_protocol(), port)) if asynchronous: thread = FuncThread(method, port, quiet=True) thread.start() TMP_THREADS.append(thread) return thread else: method(port)
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={}, endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]): env = aws_stack.get_environment(env) # decide which credentials provider to use credentialsProvider = None if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and ('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = 'com.atlassian.DefaultSTSAssumeRoleSessionCredentialsProvider' # pass through env variables to child process for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if env.region == REGION_LOCAL: # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars['AWS_CBOR_DISABLE'] = 'true' if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace('*', short_uid()) TMP_FILES.append(log_file) run('touch %s' % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level, 'log_prefix': 'KCL', 'log_subscribers': log_subscribers}) reader_thread.start() # construct stream info stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars) props_file = stream_info['properties_file'] # set kcl config options kwargs = { 'metricsLevel': 'NONE', 'initialPositionInStream': 'LATEST' } # set parameters for local connection if env.region == REGION_LOCAL: kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS) kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB) kwargs['kinesisProtocol'] = 'http%s' % ('s' if USE_SSL else '') kwargs['dynamodbProtocol'] = 'http%s' % ('s' if USE_SSL else '') kwargs['disableCertChecking'] = 'true' kwargs.update(configs) # create config file kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info['app_name'], credentialsProvider=credentialsProvider, **kwargs) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer
def start_local_api(name, port, method, asynchronous=False): print('Starting mock %s service in %s ports %s (recommended) and %s (deprecated)...' % ( name, get_service_protocol(), config.EDGE_PORT, port)) if asynchronous: thread = FuncThread(method, port, quiet=True) thread.start() TMP_THREADS.append(thread) return thread else: method(port)
def start_proxy(port, backend_url, update_listener, quiet=False, params={}): proxy_thread = GenericProxy(port=port, forward_url=backend_url, ssl=USE_SSL, update_listener=update_listener, quiet=quiet, params=params) proxy_thread.start() TMP_THREADS.append(proxy_thread) return proxy_thread
def do_run(cmd, asynchronous, print_output=None, env_vars={}, auto_restart=False): sys.stdout.flush() if asynchronous: if config.DEBUG and print_output is None: print_output = True outfile = subprocess.PIPE if print_output else None t = ShellCommandThread(cmd, outfile=outfile, env_vars=env_vars, auto_restart=auto_restart) t.start() TMP_THREADS.append(t) return t return run(cmd, env_vars=env_vars)
def do_run(cmd, asynchronous, print_output=None, env_vars={}): sys.stdout.flush() if asynchronous: if is_debug() and print_output is None: print_output = True outfile = subprocess.PIPE if print_output else None t = ShellCommandThread(cmd, outfile=outfile, env_vars=env_vars) t.start() TMP_THREADS.append(t) return t return run(cmd, env_vars=env_vars)
def do_start_thread(self) -> FuncThread: cmd = self._create_shell_command() LOG.debug("starting kinesis process %s", cmd) t = ShellCommandThread( cmd, strip_color=True, log_listener=self._log_listener, auto_restart=True, ) TMP_THREADS.append(t) t.start() return t
def do_run(cmd, asynchronous, print_output=False): sys.stdout.flush() if asynchronous: if is_debug(): print_output = True outfile = subprocess.PIPE if print_output else None t = ShellCommandThread(cmd, outfile=outfile) t.start() TMP_THREADS.append(t) return t else: return run(cmd)
def start_edge(port=None, use_ssl=True, asynchronous=False): if not port: port = config.EDGE_PORT if config.EDGE_PORT_HTTP: do_start_edge( config.EDGE_BIND_HOST, config.EDGE_PORT_HTTP, use_ssl=False, asynchronous=True, ) if port > 1024 or is_root(): return do_start_edge(config.EDGE_BIND_HOST, port, use_ssl, asynchronous=asynchronous) # process requires priviledged port but we're not root -> try running as sudo class Terminator(object): def stop(self, quiet=True): try: url = "http%s://%s:%s" % ("s" if use_ssl else "", LOCALHOST, port) requests.verify_ssl = False requests.post(url, headers={HEADER_KILL_SIGNAL: "kill"}) except Exception: pass # make sure we can run sudo commands try: ensure_can_use_sudo() except Exception as e: LOG.error("cannot start edge proxy on privileged port %s: %s", port, str(e)) return # register a signal handler to terminate the sudo process later on TMP_THREADS.append(Terminator()) # start the process as sudo sudo_cmd = "sudo " python_cmd = sys.executable cmd = "%sPYTHONPATH=.:%s %s %s %s" % ( sudo_cmd, LOCALSTACK_ROOT_FOLDER, python_cmd, __file__, port, ) process = run(cmd, asynchronous=asynchronous) return process
def start_api_server_locally(request): api = request.get('api') port = request.get('port') if api in API_SERVERS: return API_SERVERS[api] result = API_SERVERS[api] = {} def thread_func(params): return moto_main([api, '-p', str(port), '-H', constants.BIND_HOST]) thread = FuncThread(thread_func) thread.start() TMP_THREADS.append(thread) result['port'] = port result['thread'] = thread return result
def start_server_process(port): if '__server__' in API_SERVERS: return port = port or MULTI_SERVER_PORT API_SERVERS['__server__'] = config = {'port': port} LOG.info('Starting multi API server process on port %s' % port) cmd = '"%s" "%s" %s' % (sys.executable, __file__, port) env_vars = { 'PYTHONPATH': '.:%s' % constants.LOCALSTACK_ROOT_FOLDER } thread = ShellCommandThread(cmd, outfile=subprocess.PIPE, env_vars=env_vars, inherit_cwd=True) thread.start() TMP_THREADS.append(thread) config['thread'] = thread wait_for_port_open(port, retries=10, sleep_time=1) return thread
def start_server_process(port): if "__server__" in API_SERVERS: return API_SERVERS["__server__"]["thread"] port = port or get_multi_server_port() API_SERVERS["__server__"] = config = {"port": port} LOG.info("Starting multi API server process on port %s" % port) if RUN_SERVER_IN_PROCESS: cmd = '"%s" "%s" %s' % (sys.executable, __file__, port) env_vars = {"PYTHONPATH": ".:%s" % constants.LOCALSTACK_ROOT_FOLDER} thread = ShellCommandThread(cmd, outfile=subprocess.PIPE, env_vars=env_vars, inherit_cwd=True) thread.start() else: thread = start_server(port, asynchronous=True) TMP_THREADS.append(thread) config["thread"] = thread wait_for_port_open(port, retries=20, sleep_time=1) return thread
def start_api_server_locally(request): if localstack_config.FORWARD_EDGE_INMEM: if '__started__' in API_SERVERS: return API_SERVERS['__started__'] = True api = request.get('api') port = request.get('port') if api in API_SERVERS: return API_SERVERS[api] result = API_SERVERS[api] = {} def thread_func(params): if localstack_config.FORWARD_EDGE_INMEM: return moto_server.main(['-p', str(port), '-H', constants.BIND_HOST]) return moto_server.main([api, '-p', str(port), '-H', constants.BIND_HOST]) thread = FuncThread(thread_func) thread.start() TMP_THREADS.append(thread) result['port'] = port result['thread'] = thread return result
def do_run( cmd: Union[str, List], asynchronous: bool, print_output: bool = None, env_vars: Dict[str, str] = {}, auto_restart=False, strip_color: bool = False, ): sys.stdout.flush() if asynchronous: if config.DEBUG and print_output is None: print_output = True outfile = subprocess.PIPE if print_output else None t = ShellCommandThread( cmd, outfile=outfile, env_vars=env_vars, auto_restart=auto_restart, strip_color=strip_color, ) t.start() TMP_THREADS.append(t) return t return run(cmd, env_vars=env_vars)
SIGNAL_HANDLERS_SETUP = True def is_debug(): return os.environ.get('DEBUG', '').strip() not in ['', '0', 'false'] def do_run(cmd, async, print_output=False): sys.stdout.flush() if async: if is_debug(): print_output = True outfile = subprocess.PIPE if print_output else None t = ShellCommandThread(cmd, outfile=outfile) t.start() TMP_THREADS.append(t) return t else: return run(cmd) def start_proxy_for_service(service_name, port, default_backend_port, update_listener, quiet=False, params={}): # check if we have a custom backend configured custom_backend_url = os.environ.get('%s_BACKEND' % service_name.upper()) backend_url = custom_backend_url or ('http://%s:%s' % (DEFAULT_BACKEND_HOST, default_backend_port)) return start_proxy(port, backend_url=backend_url, update_listener=update_listener, quiet=quiet, params=params) def start_proxy(port, backend_url, update_listener, quiet=False, params={}): proxy_thread = GenericProxy(port=port, forward_url=backend_url, ssl=USE_SSL, update_listener=update_listener, quiet=quiet, params=params)
def run_in_thread(): thread = ProxyThread() thread.start() TMP_THREADS.append(thread) return thread
def run_async(cls, func=None, loop=None): thread = AsyncThread(func, loop=loop) thread.start() TMP_THREADS.append(thread) return thread
thread = start_worker_thread(_run) return thread.result_future def has_idle_threads(self): if hasattr(self, '_idle_semaphore'): return self._idle_semaphore.acquire(timeout=0) num_threads = len(self._threads) return num_threads < self._max_workers # Thread pool executor for running sync functions in async context. # Note: For certain APIs like DynamoDB, we need 3x threads for each parallel request, # as during request processing the API calls out to the DynamoDB API again (recursively). # (TODO: This could potentially be improved if we move entirely to asyncio functions.) THREAD_POOL = AdaptiveThreadPool() TMP_THREADS.append(THREAD_POOL) class AsyncThread(FuncThread): def __init__(self, async_func_gen=None, loop=None): """ Pass a function that receives an event loop instance and a shutdown event, and returns an async function. """ FuncThread.__init__(self, self.run_func, None) self.async_func_gen = async_func_gen self.loop = loop self.shutdown_event = None def run_func(self, *args): loop = self.loop or ensure_event_loop() self.shutdown_event = asyncio.Event() if self.async_func_gen:
def start_kcl_client_process( stream_name, listener_script, log_file=None, env=None, configs=None, endpoint_url=None, ddb_lease_table_suffix=None, env_vars=None, region_name=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=None, ): if configs is None: configs = {} if env_vars is None: env_vars = {} if log_subscribers is None: log_subscribers = [] env = aws_stack.get_environment(env) # make sure to convert stream ARN to stream name stream_name = aws_stack.kinesis_stream_name(stream_name) # decide which credentials provider to use credentialsProvider = None if ("AWS_ASSUME_ROLE_ARN" in os.environ or "AWS_ASSUME_ROLE_ARN" in env_vars) and ("AWS_ASSUME_ROLE_SESSION_NAME" in os.environ or "AWS_ASSUME_ROLE_SESSION_NAME" in env_vars): # use special credentials provider that can assume IAM roles and handle temporary STS auth tokens credentialsProvider = "cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider" # pass through env variables to child process for var_name in [ "AWS_ASSUME_ROLE_ARN", "AWS_ASSUME_ROLE_SESSION_NAME", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", ]: if var_name in os.environ and var_name not in env_vars: env_vars[var_name] = os.environ[var_name] if aws_stack.is_local_env(env): # need to disable CBOR protocol, enforce use of plain JSON, # see https://github.com/mhart/kinesalite/issues/31 env_vars["AWS_CBOR_DISABLE"] = "true" if kcl_log_level or (len(log_subscribers) > 0): if not log_file: log_file = LOG_FILE_PATTERN.replace("*", short_uid()) TMP_FILES.append(log_file) run("touch %s" % log_file) # start log output reader thread which will read the KCL log # file and print each line to stdout of this process... reader_thread = OutputReaderThread({ "file": log_file, "level": kcl_log_level, "log_prefix": "KCL", "log_subscribers": log_subscribers, }) reader_thread.start() # construct stream info stream_info = get_stream_info( stream_name, log_file, env=env, endpoint_url=endpoint_url, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars, ) props_file = stream_info["properties_file"] # set kcl config options kwargs = {"metricsLevel": "NONE", "initialPositionInStream": "LATEST"} # set parameters for local connection if aws_stack.is_local_env(env): kwargs[ "kinesisEndpoint"] = f"{LOCALHOST}:{config.service_port('kinesis')}" kwargs[ "dynamodbEndpoint"] = f"{LOCALHOST}:{config.service_port('dynamodb')}" kwargs["kinesisProtocol"] = get_service_protocol() kwargs["dynamodbProtocol"] = get_service_protocol() kwargs["disableCertChecking"] = "true" kwargs.update(configs) # create config file kclipy_helper.create_config_file( config_file=props_file, executableName=listener_script, streamName=stream_name, applicationName=stream_info["app_name"], credentialsProvider=credentialsProvider, region_name=region_name, **kwargs, ) TMP_FILES.append(props_file) # start stream consumer stream = KinesisStream(id=stream_name, params=stream_info) thread_consumer = KinesisProcessorThread.start_consumer(stream) TMP_THREADS.append(thread_consumer) return thread_consumer