def create_kinesis_stream(stream_name, shards=1, env=None, delete=False):
env = get_environment(env)
# stream
stream = KinesisStream(id=stream_name, num_shards=shards)
conn = connect_to_service('kinesis', env=env)
stream.connect(conn)
if delete:
run_safe(lambda: stream.destroy(), print_error=False)
stream.create()
stream.wait_for()
return stream
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={},
endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, region_name=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]):
env = aws_stack.get_environment(env)
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
# decide which credentials provider to use
credentialsProvider = None
if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and
('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = 'cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider'
# pass through env variables to child process
for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME',
'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if aws_stack.is_local_env(env):
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars['AWS_CBOR_DISABLE'] = 'true'
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace('*', short_uid())
TMP_FILES.append(log_file)
run('touch %s' % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level,
'log_prefix': 'KCL', 'log_subscribers': log_subscribers})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars)
props_file = stream_info['properties_file']
# set kcl config options
kwargs = {
'metricsLevel': 'NONE',
'initialPositionInStream': 'LATEST'
}
# set parameters for local connection
if aws_stack.is_local_env(env):
kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS)
kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB)
kwargs['kinesisProtocol'] = get_service_protocol()
kwargs['dynamodbProtocol'] = get_service_protocol()
kwargs['disableCertChecking'] = 'true'
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script,
streamName=stream_name, applicationName=stream_info['app_name'],
credentialsProvider=credentialsProvider, region_name=region_name, **kwargs)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
def get_kinesis_streams(filter=".*", pool={}, env=None, region=None):
if MOCK_OBJ:
return []
result = []
try:
kinesis_client = _connect("kinesis", region=region)
out = kinesis_client.list_streams()
for name in out["StreamNames"]:
if re.match(filter, name):
details = kinesis_client.describe_stream(StreamName=name)
arn = details["StreamDescription"]["StreamARN"]
stream = KinesisStream(arn)
pool[arn] = stream
stream.shards = get_kinesis_shards(stream_details=details, env=env, region=region)
result.append(stream)
except Exception:
pass
return result
def create_kinesis_stream(stream_name, shards=1, env=None, delete=False):
env = get_environment(env)
# stream
stream = KinesisStream(id=stream_name, num_shards=shards)
conn = connect_to_service('kinesis', env=env)
stream.connect(conn)
if delete:
run_safe(lambda: stream.destroy(), print_error=False)
stream.create()
stream.wait_for()
return stream
def get_kinesis_streams(filter='.*', pool={}, env=None):
if MOCK_OBJ:
return []
result = []
try:
out = cmd_kinesis('list-streams', env)
out = json.loads(out)
for name in out['StreamNames']:
if re.match(filter, name):
details = cmd_kinesis('describe-stream --stream-name %s' % name, env=env)
details = json.loads(details)
arn = details['StreamDescription']['StreamARN']
stream = KinesisStream(arn)
pool[arn] = stream
stream.shards = get_kinesis_shards(stream_details=details, env=env)
result.append(stream)
except socket.error:
pass
return result
def get_kinesis_streams(filter='.*', pool={}, env=None):
if MOCK_OBJ:
return []
result = []
try:
out = cmd_kinesis('list-streams', env)
out = json.loads(out)
for name in out['StreamNames']:
if re.match(filter, name):
details = cmd_kinesis('describe-stream --stream-name %s' % name, env=env)
details = json.loads(details)
arn = details['StreamDescription']['StreamARN']
stream = KinesisStream(arn)
pool[arn] = stream
stream.shards = get_kinesis_shards(stream_details=details, env=env)
result.append(stream)
except socket.error:
pass
return result
def get_kinesis_streams(filter='.*', pool={}, env=None):
if MOCK_OBJ:
return []
result = []
try:
kinesis_client = aws_stack.connect_to_service('kinesis')
out = kinesis_client.list_streams()
for name in out['StreamNames']:
if re.match(filter, name):
details = kinesis_client.describe_stream(StreamArn=name)
arn = details['StreamDescription']['StreamARN']
stream = KinesisStream(arn)
pool[arn] = stream
stream.shards = get_kinesis_shards(stream_details=details,
env=env)
result.append(stream)
except Exception:
pass
return result
def create_kinesis_stream(stream_name, shards=1, env=None, delete=False):
env = get_environment(env)
stream = KinesisStream(id=stream_name, num_shards=shards)
conn = connect_to_service("kinesis", env=env)
stream.connect(conn)
if delete:
run_safe(lambda: stream.destroy(), print_error=False)
stream.create()
# Note: Returning the stream without awaiting its creation (via wait_for()) to avoid API call timeouts/retries.
return stream
def wait_for_stream_created(table_name):
stream_name = get_kinesis_stream_name(table_name)
stream = KinesisStream(id=stream_name, num_shards=1)
kinesis = aws_stack.connect_to_service('kinesis',
env=get_environment(None))
stream.connect(kinesis)
stream.wait_for()
def wait_for_stream_created(table_name):
stream_name = get_kinesis_stream_name(table_name)
stream = KinesisStream(id=stream_name, num_shards=1)
kinesis = aws_stack.create_external_boto_client(
"kinesis", env=get_environment(None))
stream.connect(kinesis)
stream.wait_for()
def start_kcl_client_process(
stream_name,
listener_script,
log_file=None,
env=None,
configs=None,
endpoint_url=None,
ddb_lease_table_suffix=None,
env_vars=None,
region_name=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL,
log_subscribers=None,
):
if configs is None:
configs = {}
if env_vars is None:
env_vars = {}
if log_subscribers is None:
log_subscribers = []
env = aws_stack.get_environment(env)
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
# decide which credentials provider to use
credentialsProvider = None
if ("AWS_ASSUME_ROLE_ARN" in os.environ or "AWS_ASSUME_ROLE_ARN"
in env_vars) and ("AWS_ASSUME_ROLE_SESSION_NAME" in os.environ
or "AWS_ASSUME_ROLE_SESSION_NAME" in env_vars):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = "cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider"
# pass through env variables to child process
for var_name in [
"AWS_ASSUME_ROLE_ARN",
"AWS_ASSUME_ROLE_SESSION_NAME",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
]:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if aws_stack.is_local_env(env):
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars["AWS_CBOR_DISABLE"] = "true"
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace("*", short_uid())
TMP_FILES.append(log_file)
run("touch %s" % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({
"file": log_file,
"level": kcl_log_level,
"log_prefix": "KCL",
"log_subscribers": log_subscribers,
})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(
stream_name,
log_file,
env=env,
endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix,
env_vars=env_vars,
)
props_file = stream_info["properties_file"]
# set kcl config options
kwargs = {"metricsLevel": "NONE", "initialPositionInStream": "LATEST"}
# set parameters for local connection
if aws_stack.is_local_env(env):
kwargs[
"kinesisEndpoint"] = f"{LOCALHOST}:{config.service_port('kinesis')}"
kwargs[
"dynamodbEndpoint"] = f"{LOCALHOST}:{config.service_port('dynamodb')}"
kwargs["kinesisProtocol"] = config.get_protocol()
kwargs["dynamodbProtocol"] = config.get_protocol()
kwargs["disableCertChecking"] = "true"
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(
config_file=props_file,
executableName=listener_script,
streamName=stream_name,
applicationName=stream_info["app_name"],
credentialsProvider=credentialsProvider,
region_name=region_name,
**kwargs,
)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
