def start_proxy(port, backend_url, update_listener=None, quiet=False, params={}, use_ssl=None):
use_ssl = config.USE_SSL if use_ssl is None else use_ssl
proxy_thread = GenericProxy(port=port, forward_url=backend_url,
ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def do_start_thread(self) -> FuncThread:
install.install_elasticmq()
# create config file
config_params = config_template % (
config.LOCALSTACK_HOSTNAME,
get_free_tcp_port(),
self.port,
)
# create temporary config
config_file = os.path.join(config.dirs.tmp, "sqs.%s.conf" % short_uid())
LOG.debug("saving config file to %s:\n%s", config_file, config_params)
TMP_FILES.append(config_file)
save_file(config_file, config_params)
# start process
cmd = [
"java",
f"-Dconfig.file={config_file}",
f"-Xmx{self.max_heap_size}",
"-jar",
os.path.join(self.install_dir, "elasticmq-server.jar"),
]
LOG.debug("starting elasticmq server with command %s", " ".join(cmd))
t = ShellCommandThread(
cmd,
strip_color=True,
log_listener=self._log_listener,
auto_restart=True,
)
TMP_THREADS.append(t)
t.start()
return t
def start_api_server_locally(request):
if localstack_config.FORWARD_EDGE_INMEM:
if "__started__" in API_SERVERS:
return
API_SERVERS["__started__"] = True
api = request.get("api")
port = request.get("port")
if api in API_SERVERS:
return API_SERVERS[api]
result = API_SERVERS[api] = {}
def thread_func(params):
if localstack_config.FORWARD_EDGE_INMEM:
return moto_server.main(
["-p", str(port), "-H", constants.BIND_HOST])
return moto_server.main(
[api, "-p", str(port), "-H", constants.BIND_HOST])
thread = FuncThread(thread_func)
thread.start()
TMP_THREADS.append(thread)
result["port"] = port
result["thread"] = thread
return result
def start_ssl_proxy(
port: int,
target: PortOrUrl,
target_ssl=False,
client_cert_key: Tuple[str, str] = None,
asynchronous: bool = False,
fix_encoding: bool = False,
):
"""Start a proxy server that accepts SSL requests and forwards requests to a backend (either SSL or non-SSL)"""
if client_cert_key or fix_encoding:
# use a custom proxy listener, in case the user provides client certificates for authentication
if client_cert_key:
server = _do_start_ssl_proxy_with_client_auth(
port, target, client_cert_key=client_cert_key)
else:
server = _do_start_ssl_proxy_with_listener(port, target)
if not asynchronous:
server.join()
return server
def _run(*args):
return _do_start_ssl_proxy(port, target, target_ssl=target_ssl)
if not asynchronous:
return _run()
proxy = FuncThread(_run)
TMP_THREADS.append(proxy)
proxy.start()
return proxy
def start_edge(port=None, use_ssl=True, asynchronous=False):
if not port:
port = config.EDGE_PORT
if config.EDGE_PORT_HTTP:
do_start_edge(config.EDGE_PORT_HTTP, use_ssl=False, asynchronous=True)
if port > 1024 or is_root():
return do_start_edge(port, use_ssl, asynchronous=asynchronous)
# process requires priviledged port but we're not root -> try running as sudo
class Terminator(object):
def stop(self, quiet=True):
try:
url = 'http%s://localhost:%s' % ('s' if use_ssl else '', port)
requests.verify_ssl = False
requests.post(url, headers={HEADER_KILL_SIGNAL: 'kill'})
except Exception:
pass
# make sure we can run sudo commands
ensure_can_use_sudo()
# register a signal handler to terminate the sudo process later on
TMP_THREADS.append(Terminator())
# start the process as sudo
sudo_cmd = 'sudo '
python_cmd = sys.executable
cmd = '%sPYTHONPATH=.:%s %s %s %s' % (sudo_cmd, LOCALSTACK_ROOT_FOLDER,
python_cmd, __file__, port)
process = run(cmd, asynchronous=asynchronous)
return process
def start_edge(port=None, use_ssl=True, asynchronous=False):
if not port:
port = config.EDGE_PORT
if config.EDGE_PORT_HTTP and config.EDGE_PORT_HTTP != port:
do_start_edge(
config.EDGE_BIND_HOST,
config.EDGE_PORT_HTTP,
use_ssl=False,
asynchronous=True,
)
if port > 1024 or is_root():
return do_start_edge(config.EDGE_BIND_HOST,
port,
use_ssl,
asynchronous=asynchronous)
# process requires privileged port but we're not root -> try running as sudo
class Terminator(object):
def stop(self, quiet=True):
try:
url = "http%s://%s:%s" % ("s" if use_ssl else "", LOCALHOST,
port)
requests.verify_ssl = False
requests.post(url, headers={HEADER_KILL_SIGNAL: "kill"})
except Exception:
pass
# register a signal handler to terminate the sudo process later on
TMP_THREADS.append(Terminator())
return run_process_as_sudo("edge", port, asynchronous=asynchronous)
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={},
endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, region_name=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]):
env = aws_stack.get_environment(env)
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
# decide which credentials provider to use
credentialsProvider = None
if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and
('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = 'cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider'
# pass through env variables to child process
for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME',
'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if aws_stack.is_local_env(env):
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars['AWS_CBOR_DISABLE'] = 'true'
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace('*', short_uid())
TMP_FILES.append(log_file)
run('touch %s' % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level,
'log_prefix': 'KCL', 'log_subscribers': log_subscribers})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars)
props_file = stream_info['properties_file']
# set kcl config options
kwargs = {
'metricsLevel': 'NONE',
'initialPositionInStream': 'LATEST'
}
# set parameters for local connection
if aws_stack.is_local_env(env):
kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS)
kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB)
kwargs['kinesisProtocol'] = get_service_protocol()
kwargs['dynamodbProtocol'] = get_service_protocol()
kwargs['disableCertChecking'] = 'true'
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script,
streamName=stream_name, applicationName=stream_info['app_name'],
credentialsProvider=credentialsProvider, region_name=region_name, **kwargs)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
def start_proxy_server(port, forward_url=None, use_ssl=None, update_listener=None, quiet=False, params={}):
if USE_HTTP2_SERVER:
return start_proxy_server_http2(port=port, forward_url=forward_url,
use_ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params)
proxy_thread = GenericProxy(port=port, forward_url=forward_url,
ssl=use_ssl, update_listener=update_listener, quiet=quiet, params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def start_local_api(name, port, method, asynchronous=False):
print('Starting mock %s service (%s port %s)...' % (name, get_service_protocol(), port))
if asynchronous:
thread = FuncThread(method, port, quiet=True)
thread.start()
TMP_THREADS.append(thread)
return thread
else:
method(port)
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={},
endpoint_url=None, ddb_lease_table_suffix=None, env_vars={},
kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]):
env = aws_stack.get_environment(env)
# decide which credentials provider to use
credentialsProvider = None
if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and
('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = 'com.atlassian.DefaultSTSAssumeRoleSessionCredentialsProvider'
# pass through env variables to child process
for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME',
'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if env.region == REGION_LOCAL:
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars['AWS_CBOR_DISABLE'] = 'true'
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace('*', short_uid())
TMP_FILES.append(log_file)
run('touch %s' % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level,
'log_prefix': 'KCL', 'log_subscribers': log_subscribers})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars)
props_file = stream_info['properties_file']
# set kcl config options
kwargs = {
'metricsLevel': 'NONE',
'initialPositionInStream': 'LATEST'
}
# set parameters for local connection
if env.region == REGION_LOCAL:
kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS)
kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB)
kwargs['kinesisProtocol'] = 'http%s' % ('s' if USE_SSL else '')
kwargs['dynamodbProtocol'] = 'http%s' % ('s' if USE_SSL else '')
kwargs['disableCertChecking'] = 'true'
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script,
streamName=stream_name, applicationName=stream_info['app_name'],
credentialsProvider=credentialsProvider, **kwargs)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
def start_local_api(name, port, method, asynchronous=False):
print('Starting mock %s service in %s ports %s (recommended) and %s (deprecated)...' % (
name, get_service_protocol(), config.EDGE_PORT, port))
if asynchronous:
thread = FuncThread(method, port, quiet=True)
thread.start()
TMP_THREADS.append(thread)
return thread
else:
method(port)
def start_proxy(port, backend_url, update_listener, quiet=False, params={}):
proxy_thread = GenericProxy(port=port,
forward_url=backend_url,
ssl=USE_SSL,
update_listener=update_listener,
quiet=quiet,
params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def do_run(cmd, asynchronous, print_output=None, env_vars={}, auto_restart=False):
sys.stdout.flush()
if asynchronous:
if config.DEBUG and print_output is None:
print_output = True
outfile = subprocess.PIPE if print_output else None
t = ShellCommandThread(cmd, outfile=outfile, env_vars=env_vars, auto_restart=auto_restart)
t.start()
TMP_THREADS.append(t)
return t
return run(cmd, env_vars=env_vars)
def do_run(cmd, asynchronous, print_output=None, env_vars={}):
sys.stdout.flush()
if asynchronous:
if is_debug() and print_output is None:
print_output = True
outfile = subprocess.PIPE if print_output else None
t = ShellCommandThread(cmd, outfile=outfile, env_vars=env_vars)
t.start()
TMP_THREADS.append(t)
return t
return run(cmd, env_vars=env_vars)
def do_start_thread(self) -> FuncThread:
cmd = self._create_shell_command()
LOG.debug("starting kinesis process %s", cmd)
t = ShellCommandThread(
cmd,
strip_color=True,
log_listener=self._log_listener,
auto_restart=True,
)
TMP_THREADS.append(t)
t.start()
return t
def do_run(cmd, asynchronous, print_output=False):
sys.stdout.flush()
if asynchronous:
if is_debug():
print_output = True
outfile = subprocess.PIPE if print_output else None
t = ShellCommandThread(cmd, outfile=outfile)
t.start()
TMP_THREADS.append(t)
return t
else:
return run(cmd)
def start_edge(port=None, use_ssl=True, asynchronous=False):
if not port:
port = config.EDGE_PORT
if config.EDGE_PORT_HTTP:
do_start_edge(
config.EDGE_BIND_HOST,
config.EDGE_PORT_HTTP,
use_ssl=False,
asynchronous=True,
)
if port > 1024 or is_root():
return do_start_edge(config.EDGE_BIND_HOST,
port,
use_ssl,
asynchronous=asynchronous)
# process requires priviledged port but we're not root -> try running as sudo
class Terminator(object):
def stop(self, quiet=True):
try:
url = "http%s://%s:%s" % ("s" if use_ssl else "", LOCALHOST,
port)
requests.verify_ssl = False
requests.post(url, headers={HEADER_KILL_SIGNAL: "kill"})
except Exception:
pass
# make sure we can run sudo commands
try:
ensure_can_use_sudo()
except Exception as e:
LOG.error("cannot start edge proxy on privileged port %s: %s", port,
str(e))
return
# register a signal handler to terminate the sudo process later on
TMP_THREADS.append(Terminator())
# start the process as sudo
sudo_cmd = "sudo "
python_cmd = sys.executable
cmd = "%sPYTHONPATH=.:%s %s %s %s" % (
sudo_cmd,
LOCALSTACK_ROOT_FOLDER,
python_cmd,
__file__,
port,
)
process = run(cmd, asynchronous=asynchronous)
return process
def start_api_server_locally(request):
api = request.get('api')
port = request.get('port')
if api in API_SERVERS:
return API_SERVERS[api]
result = API_SERVERS[api] = {}
def thread_func(params):
return moto_main([api, '-p', str(port), '-H', constants.BIND_HOST])
thread = FuncThread(thread_func)
thread.start()
TMP_THREADS.append(thread)
result['port'] = port
result['thread'] = thread
return result
def start_server_process(port):
if '__server__' in API_SERVERS:
return
port = port or MULTI_SERVER_PORT
API_SERVERS['__server__'] = config = {'port': port}
LOG.info('Starting multi API server process on port %s' % port)
cmd = '"%s" "%s" %s' % (sys.executable, __file__, port)
env_vars = {
'PYTHONPATH': '.:%s' % constants.LOCALSTACK_ROOT_FOLDER
}
thread = ShellCommandThread(cmd, outfile=subprocess.PIPE, env_vars=env_vars,
inherit_cwd=True)
thread.start()
TMP_THREADS.append(thread)
config['thread'] = thread
wait_for_port_open(port, retries=10, sleep_time=1)
return thread
def start_server_process(port):
if "__server__" in API_SERVERS:
return API_SERVERS["__server__"]["thread"]
port = port or get_multi_server_port()
API_SERVERS["__server__"] = config = {"port": port}
LOG.info("Starting multi API server process on port %s" % port)
if RUN_SERVER_IN_PROCESS:
cmd = '"%s" "%s" %s' % (sys.executable, __file__, port)
env_vars = {"PYTHONPATH": ".:%s" % constants.LOCALSTACK_ROOT_FOLDER}
thread = ShellCommandThread(cmd,
outfile=subprocess.PIPE,
env_vars=env_vars,
inherit_cwd=True)
thread.start()
else:
thread = start_server(port, asynchronous=True)
TMP_THREADS.append(thread)
config["thread"] = thread
wait_for_port_open(port, retries=20, sleep_time=1)
return thread
def start_api_server_locally(request):
if localstack_config.FORWARD_EDGE_INMEM:
if '__started__' in API_SERVERS:
return
API_SERVERS['__started__'] = True
api = request.get('api')
port = request.get('port')
if api in API_SERVERS:
return API_SERVERS[api]
result = API_SERVERS[api] = {}
def thread_func(params):
if localstack_config.FORWARD_EDGE_INMEM:
return moto_server.main(['-p', str(port), '-H', constants.BIND_HOST])
return moto_server.main([api, '-p', str(port), '-H', constants.BIND_HOST])
thread = FuncThread(thread_func)
thread.start()
TMP_THREADS.append(thread)
result['port'] = port
result['thread'] = thread
return result
def do_run(
cmd: Union[str, List],
asynchronous: bool,
print_output: bool = None,
env_vars: Dict[str, str] = {},
auto_restart=False,
strip_color: bool = False,
):
sys.stdout.flush()
if asynchronous:
if config.DEBUG and print_output is None:
print_output = True
outfile = subprocess.PIPE if print_output else None
t = ShellCommandThread(
cmd,
outfile=outfile,
env_vars=env_vars,
auto_restart=auto_restart,
strip_color=strip_color,
)
t.start()
TMP_THREADS.append(t)
return t
return run(cmd, env_vars=env_vars)
SIGNAL_HANDLERS_SETUP = True
def is_debug():
return os.environ.get('DEBUG', '').strip() not in ['', '0', 'false']
def do_run(cmd, async, print_output=False):
sys.stdout.flush()
if async:
if is_debug():
print_output = True
outfile = subprocess.PIPE if print_output else None
t = ShellCommandThread(cmd, outfile=outfile)
t.start()
TMP_THREADS.append(t)
return t
else:
return run(cmd)
def start_proxy_for_service(service_name, port, default_backend_port, update_listener, quiet=False, params={}):
# check if we have a custom backend configured
custom_backend_url = os.environ.get('%s_BACKEND' % service_name.upper())
backend_url = custom_backend_url or ('http://%s:%s' % (DEFAULT_BACKEND_HOST, default_backend_port))
return start_proxy(port, backend_url=backend_url, update_listener=update_listener, quiet=quiet, params=params)
def start_proxy(port, backend_url, update_listener, quiet=False, params={}):
proxy_thread = GenericProxy(port=port, forward_url=backend_url,
ssl=USE_SSL, update_listener=update_listener, quiet=quiet, params=params)
def run_in_thread():
thread = ProxyThread()
thread.start()
TMP_THREADS.append(thread)
return thread
SIGNAL_HANDLERS_SETUP = True
def is_debug():
return os.environ.get('DEBUG', '').strip() not in ['', '0', 'false']
def do_run(cmd, async, print_output=False):
sys.stdout.flush()
if async:
if is_debug():
print_output = True
outfile = subprocess.PIPE if print_output else None
t = ShellCommandThread(cmd, outfile=outfile)
t.start()
TMP_THREADS.append(t)
return t
else:
return run(cmd)
def start_proxy_for_service(service_name, port, default_backend_port, update_listener, quiet=False, params={}):
# check if we have a custom backend configured
custom_backend_url = os.environ.get('%s_BACKEND' % service_name.upper())
backend_url = custom_backend_url or ('http://%s:%s' % (DEFAULT_BACKEND_HOST, default_backend_port))
return start_proxy(port, backend_url=backend_url, update_listener=update_listener, quiet=quiet, params=params)
def start_proxy(port, backend_url, update_listener, quiet=False, params={}):
proxy_thread = GenericProxy(port=port, forward_url=backend_url,
ssl=USE_SSL, update_listener=update_listener, quiet=quiet, params=params)
def start_proxy(port, backend_url, update_listener, quiet=False, params={}):
proxy_thread = GenericProxy(port=port, forward_url=backend_url,
ssl=USE_SSL, update_listener=update_listener, quiet=quiet, params=params)
proxy_thread.start()
TMP_THREADS.append(proxy_thread)
return proxy_thread
def run_async(cls, func=None, loop=None):
thread = AsyncThread(func, loop=loop)
thread.start()
TMP_THREADS.append(thread)
return thread
thread = start_worker_thread(_run)
return thread.result_future
def has_idle_threads(self):
if hasattr(self, '_idle_semaphore'):
return self._idle_semaphore.acquire(timeout=0)
num_threads = len(self._threads)
return num_threads < self._max_workers
# Thread pool executor for running sync functions in async context.
# Note: For certain APIs like DynamoDB, we need 3x threads for each parallel request,
# as during request processing the API calls out to the DynamoDB API again (recursively).
# (TODO: This could potentially be improved if we move entirely to asyncio functions.)
THREAD_POOL = AdaptiveThreadPool()
TMP_THREADS.append(THREAD_POOL)
class AsyncThread(FuncThread):
def __init__(self, async_func_gen=None, loop=None):
""" Pass a function that receives an event loop instance and a shutdown event,
and returns an async function. """
FuncThread.__init__(self, self.run_func, None)
self.async_func_gen = async_func_gen
self.loop = loop
self.shutdown_event = None
def run_func(self, *args):
loop = self.loop or ensure_event_loop()
self.shutdown_event = asyncio.Event()
if self.async_func_gen:
def start_kcl_client_process(
stream_name,
listener_script,
log_file=None,
env=None,
configs=None,
endpoint_url=None,
ddb_lease_table_suffix=None,
env_vars=None,
region_name=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL,
log_subscribers=None,
):
if configs is None:
configs = {}
if env_vars is None:
env_vars = {}
if log_subscribers is None:
log_subscribers = []
env = aws_stack.get_environment(env)
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
# decide which credentials provider to use
credentialsProvider = None
if ("AWS_ASSUME_ROLE_ARN" in os.environ or "AWS_ASSUME_ROLE_ARN"
in env_vars) and ("AWS_ASSUME_ROLE_SESSION_NAME" in os.environ
or "AWS_ASSUME_ROLE_SESSION_NAME" in env_vars):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = "cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider"
# pass through env variables to child process
for var_name in [
"AWS_ASSUME_ROLE_ARN",
"AWS_ASSUME_ROLE_SESSION_NAME",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
]:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if aws_stack.is_local_env(env):
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars["AWS_CBOR_DISABLE"] = "true"
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace("*", short_uid())
TMP_FILES.append(log_file)
run("touch %s" % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({
"file": log_file,
"level": kcl_log_level,
"log_prefix": "KCL",
"log_subscribers": log_subscribers,
})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(
stream_name,
log_file,
env=env,
endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix,
env_vars=env_vars,
)
props_file = stream_info["properties_file"]
# set kcl config options
kwargs = {"metricsLevel": "NONE", "initialPositionInStream": "LATEST"}
# set parameters for local connection
if aws_stack.is_local_env(env):
kwargs[
"kinesisEndpoint"] = f"{LOCALHOST}:{config.service_port('kinesis')}"
kwargs[
"dynamodbEndpoint"] = f"{LOCALHOST}:{config.service_port('dynamodb')}"
kwargs["kinesisProtocol"] = get_service_protocol()
kwargs["dynamodbProtocol"] = get_service_protocol()
kwargs["disableCertChecking"] = "true"
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(
config_file=props_file,
executableName=listener_script,
streamName=stream_name,
applicationName=stream_info["app_name"],
credentialsProvider=credentialsProvider,
region_name=region_name,
**kwargs,
)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
