def start_kinesis_mock(port=None, asynchronous=False, update_listener=None):
kinesis_mock_bin = install.install_kinesis_mock()
backend_port = get_free_tcp_port()
global PORT_KINESIS_BACKEND
PORT_KINESIS_BACKEND = backend_port
kinesis_data_dir_param = ""
if config.DATA_DIR:
kinesis_data_dir = "%s/kinesis" % config.DATA_DIR
mkdir(kinesis_data_dir)
kinesis_data_dir_param = "SHOULD_PERSIST_DATA=true PERSIST_PATH=%s" % kinesis_data_dir
if not config.LS_LOG:
log_level = "INFO"
elif config.LS_LOG == "warning":
log_level = "WARN"
else:
log_level = config.LS_LOG.upper()
log_level_param = "LOG_LEVEL=%s" % log_level
latency = config.KINESIS_LATENCY + "ms"
latency_param = (
"CREATE_STREAM_DURATION={l} DELETE_STREAM_DURATION={l} REGISTER_STREAM_CONSUMER_DURATION={l} "
"START_STREAM_ENCRYPTION_DURATION={l} STOP_STREAM_ENCRYPTION_DURATION={l} "
"DEREGISTER_STREAM_CONSUMER_DURATION={l} MERGE_SHARDS_DURATION={l} SPLIT_SHARD_DURATION={l} "
"UPDATE_SHARD_COUNT_DURATION={l}").format(l=latency)
if config.KINESIS_INITIALIZE_STREAMS != "":
initialize_streams_param = "INITIALIZE_STREAMS=%s" % config.KINESIS_INITIALIZE_STREAMS
else:
initialize_streams_param = ""
if kinesis_mock_bin.endswith(".jar"):
cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s" % (
backend_port,
config.KINESIS_SHARD_LIMIT,
latency_param,
kinesis_data_dir_param,
log_level_param,
initialize_streams_param,
kinesis_mock_bin,
)
else:
chmod_r(kinesis_mock_bin, 0o777)
cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1" % (
backend_port,
config.KINESIS_SHARD_LIMIT,
latency_param,
kinesis_data_dir_param,
log_level_param,
initialize_streams_param,
kinesis_mock_bin,
)
return _run_proxy_and_command(
cmd=cmd,
port=port,
backend_port=backend_port,
update_listener=update_listener,
asynchronous=asynchronous,
)
def create_kinesis_mock_server(port=None,
persist_path: Optional[str] = None
) -> KinesisMockServer:
"""
Creates a new Kinesis Mock server instance. Installs Kinesis Mock on the host first if necessary.
Introspects on the host config to determine server configuration:
config.dirs.data -> if set, the server runs with persistence using the path to store data
config.LS_LOG -> configure kinesis mock log level (defaults to INFO)
config.KINESIS_LATENCY -> configure stream latency (in milliseconds)
config.KINESIS_INITIALIZE_STREAMS -> Initialize the given streams on startup
"""
port = port or get_free_tcp_port()
is_kinesis_mock_installed, kinesis_mock_bin_path = install.get_is_kinesis_mock_installed(
)
if not is_kinesis_mock_installed:
install.install_kinesis_mock(kinesis_mock_bin_path)
persist_path = (f"{config.dirs.data}/kinesis"
if not persist_path and config.dirs.data else persist_path)
if persist_path:
mkdir(persist_path)
if config.LS_LOG:
if config.LS_LOG == "warning":
log_level = "WARN"
else:
log_level = config.LS_LOG.upper()
else:
log_level = "INFO"
latency = config.KINESIS_LATENCY + "ms"
initialize_streams = (config.KINESIS_INITIALIZE_STREAMS
if config.KINESIS_INITIALIZE_STREAMS else None)
server = KinesisMockServer(
port=port,
bin_path=kinesis_mock_bin_path,
log_level=log_level,
latency=latency,
initialize_streams=initialize_streams,
data_dir=persist_path,
)
return server
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None):
kinesis_mock_bin = install.install_kinesis_mock()
port = port or config.PORT_KINESIS
backend_port = get_free_tcp_port()
kinesis_data_dir_param = ''
if config.DATA_DIR:
kinesis_data_dir = '%s/kinesis' % config.DATA_DIR
mkdir(kinesis_data_dir)
# FIXME: workaround for https://github.com/localstack/localstack/issues/4227
streams_file = os.path.join(kinesis_data_dir, 'kinesis-data.json')
if not os.path.exists(streams_file):
with open(streams_file, 'w') as fd:
fd.write('{"streams":{}}')
kinesis_data_dir_param = 'SHOULD_PERSIST_DATA=true PERSIST_PATH=%s' % kinesis_data_dir
if not config.LS_LOG:
log_level = 'INFO'
elif config.LS_LOG == 'warning':
log_level = 'WARN'
else:
log_level = config.LS_LOG.upper()
log_level_param = 'LOG_LEVEL=%s' % log_level
latency = config.KINESIS_LATENCY + 'ms'
latency_param = 'CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s ' \
'START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s ' \
'DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s ' \
'UPDATE_SHARD_COUNT_DURATION=%s' \
% (latency, latency, latency, latency, latency, latency, latency, latency, latency)
if config.KINESIS_INITIALIZE_STREAMS != '':
initialize_streams_param = 'INITIALIZE_STREAMS=%s' % (
config.KINESIS_INITIALIZE_STREAMS)
else:
initialize_streams_param = ''
if kinesis_mock_bin.endswith('.jar'):
cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s' \
% (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param,
log_level_param, initialize_streams_param, kinesis_mock_bin)
else:
chmod_r(kinesis_mock_bin, 0o777)
cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1' \
% (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param,
log_level_param, initialize_streams_param, kinesis_mock_bin)
LOGGER.info('starting kinesis-mock proxy %d:%d with cmd: %s', port,
backend_port, cmd)
start_proxy_for_service('kinesis', port, backend_port, update_listener)
return do_run(cmd, asynchronous)
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None):
kinesis_mock_bin = install.install_kinesis_mock()
port = port or config.PORT_KINESIS
backend_port = get_free_tcp_port()
kinesis_data_dir_param = ""
if config.DATA_DIR:
kinesis_data_dir = "%s/kinesis" % config.DATA_DIR
mkdir(kinesis_data_dir)
kinesis_data_dir_param = "SHOULD_PERSIST_DATA=true PERSIST_PATH=%s" % kinesis_data_dir
if not config.LS_LOG:
log_level = "INFO"
elif config.LS_LOG == "warning":
log_level = "WARN"
else:
log_level = config.LS_LOG.upper()
log_level_param = "LOG_LEVEL=%s" % log_level
latency = config.KINESIS_LATENCY + "ms"
latency_param = (
"CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s "
"START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s "
"DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s "
"UPDATE_SHARD_COUNT_DURATION=%s" % (
latency,
latency,
latency,
latency,
latency,
latency,
latency,
latency,
latency,
))
if config.KINESIS_INITIALIZE_STREAMS != "":
initialize_streams_param = "INITIALIZE_STREAMS=%s" % config.KINESIS_INITIALIZE_STREAMS
else:
initialize_streams_param = ""
if kinesis_mock_bin.endswith(".jar"):
cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s" % (
backend_port,
config.KINESIS_SHARD_LIMIT,
latency_param,
kinesis_data_dir_param,
log_level_param,
initialize_streams_param,
kinesis_mock_bin,
)
else:
chmod_r(kinesis_mock_bin, 0o777)
cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1" % (
backend_port,
config.KINESIS_SHARD_LIMIT,
latency_param,
kinesis_data_dir_param,
log_level_param,
initialize_streams_param,
kinesis_mock_bin,
)
LOGGER.info("starting kinesis-mock proxy %d:%d with cmd: %s", port,
backend_port, cmd)
start_proxy_for_service("kinesis", port, backend_port, update_listener)
return do_run(cmd, asynchronous)