def get_stream_info(stream_name, log_file=None, shards=None, env=None, endpoint_url=None,
ddb_lease_table_suffix=None, env_vars={}):
if not ddb_lease_table_suffix:
ddb_lease_table_suffix = DEFAULT_DDB_LEASE_TABLE_SUFFIX
# construct stream info
env = aws_stack.get_environment(env)
props_file = os.path.join(tempfile.gettempdir(), 'kclipy.%s.properties' % short_uid())
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
app_name = '%s%s' % (stream_name, ddb_lease_table_suffix)
stream_info = {
'name': stream_name,
'region': aws_stack.get_region(),
'shards': shards,
'properties_file': props_file,
'log_file': log_file,
'app_name': app_name,
'env_vars': env_vars
}
# set local connection
if aws_stack.is_local_env(env):
stream_info['conn_kwargs'] = {
'host': HOSTNAME,
'port': config.PORT_KINESIS,
'is_secure': bool(USE_SSL)
}
if endpoint_url:
if 'conn_kwargs' not in stream_info:
stream_info['conn_kwargs'] = {}
url = urlparse(endpoint_url)
stream_info['conn_kwargs']['host'] = url.hostname
stream_info['conn_kwargs']['port'] = url.port
stream_info['conn_kwargs']['is_secure'] = url.scheme == 'https'
return stream_info
def start_kcl_client_process(stream_name, listener_script, log_file=None, env=None, configs={},
endpoint_url=None, ddb_lease_table_suffix=None, env_vars={}, region_name=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=[]):
env = aws_stack.get_environment(env)
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
# decide which credentials provider to use
credentialsProvider = None
if (('AWS_ASSUME_ROLE_ARN' in os.environ or 'AWS_ASSUME_ROLE_ARN' in env_vars) and
('AWS_ASSUME_ROLE_SESSION_NAME' in os.environ or 'AWS_ASSUME_ROLE_SESSION_NAME' in env_vars)):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = 'cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider'
# pass through env variables to child process
for var_name in ['AWS_ASSUME_ROLE_ARN', 'AWS_ASSUME_ROLE_SESSION_NAME',
'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if aws_stack.is_local_env(env):
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars['AWS_CBOR_DISABLE'] = 'true'
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace('*', short_uid())
TMP_FILES.append(log_file)
run('touch %s' % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level,
'log_prefix': 'KCL', 'log_subscribers': log_subscribers})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(stream_name, log_file, env=env, endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars)
props_file = stream_info['properties_file']
# set kcl config options
kwargs = {
'metricsLevel': 'NONE',
'initialPositionInStream': 'LATEST'
}
# set parameters for local connection
if aws_stack.is_local_env(env):
kwargs['kinesisEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_KINESIS)
kwargs['dynamodbEndpoint'] = '%s:%s' % (HOSTNAME, config.PORT_DYNAMODB)
kwargs['kinesisProtocol'] = get_service_protocol()
kwargs['dynamodbProtocol'] = get_service_protocol()
kwargs['disableCertChecking'] = 'true'
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(config_file=props_file, executableName=listener_script,
streamName=stream_name, applicationName=stream_info['app_name'],
credentialsProvider=credentialsProvider, region_name=region_name, **kwargs)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
def put_log_events_model(self, log_group_name, log_stream_name, log_events,
sequence_token):
# TODO: call/patch upstream method here, instead of duplicating the code!
self.last_ingestion_time = int(unix_time_millis())
self.stored_bytes += sum(
[len(log_event["message"]) for log_event in log_events])
events = [
logs_models.LogEvent(self.last_ingestion_time, log_event)
for log_event in log_events
]
self.events += events
self.upload_sequence_token += 1
log_events = [{
"id": event.event_id,
"timestamp": event.timestamp,
"message": event.message,
} for event in events]
data = {
"messageType": "DATA_MESSAGE",
"owner": aws_stack.get_account_id(),
"logGroup": log_group_name,
"logStream": log_stream_name,
"subscriptionFilters": [self.filter_name],
"logEvents": log_events,
}
output = io.BytesIO()
with GzipFile(fileobj=output, mode="w") as f:
f.write(json.dumps(data, separators=(",", ":")).encode("utf-8"))
payload_gz_encoded = base64.b64encode(
output.getvalue()).decode("utf-8")
event = {"awslogs": {"data": payload_gz_encoded}}
if self.destination_arn:
if ":lambda:" in self.destination_arn:
client = aws_stack.connect_to_service("lambda")
lambda_name = aws_stack.lambda_function_name(
self.destination_arn)
client.invoke(FunctionName=lambda_name,
Payload=json.dumps(event))
if ":kinesis:" in self.destination_arn:
client = aws_stack.connect_to_service("kinesis")
stream_name = aws_stack.kinesis_stream_name(
self.destination_arn)
client.put_record(
StreamName=stream_name,
Data=json.dumps(payload_gz_encoded),
PartitionKey=log_group_name,
)
if ":firehose:" in self.destination_arn:
client = aws_stack.connect_to_service("firehose")
firehose_name = aws_stack.firehose_name(self.destination_arn)
client.put_record(
DeliveryStreamName=firehose_name,
Record={"Data": json.dumps(payload_gz_encoded)},
)
def put_subscription_filter(*args, **kwargs):
log_group_name = args[0]
filter_name = args[1]
filter_pattern = args[2]
destination_arn = args[3]
role_arn = args[4]
log_group = logs_models.logs_backends[
aws_stack.get_region()].groups.get(log_group_name)
if not log_group:
raise ResourceNotFoundException(
'The specified log group does not exist.')
if ':lambda:' in destination_arn:
client = aws_stack.connect_to_service('lambda')
lambda_name = aws_stack.lambda_function_name(destination_arn)
try:
client.get_function(FunctionName=lambda_name)
except Exception:
raise InvalidParameterException(
'destinationArn for vendor lambda cannot be used with roleArn'
)
elif ':kinesis:' in destination_arn:
client = aws_stack.connect_to_service('kinesis')
stream_name = aws_stack.kinesis_stream_name(destination_arn)
try:
client.describe_stream(StreamName=stream_name)
except Exception:
raise InvalidParameterException(
'Could not deliver test message to specified Kinesis stream. '
'Check if the given kinesis stream is in ACTIVE state. '
)
elif ':firehose:' in destination_arn:
client = aws_stack.connect_to_service('firehose')
firehose_name = aws_stack.firehose_name(destination_arn)
try:
client.describe_delivery_stream(
DeliveryStreamName=firehose_name)
except Exception:
raise InvalidParameterException(
'Could not deliver test message to specified Firehose stream. '
'Check if the given Firehose stream is in ACTIVE state.'
)
else:
service = aws_stack.extract_service_from_arn(destination_arn)
raise InvalidParameterException(
'PutSubscriptionFilter operation cannot work with destinationArn for vendor %s'
% service)
log_group.put_subscription_filter(filter_name, filter_pattern,
destination_arn, role_arn)
def moto_put_subscription_filter(fn, self, *args, **kwargs):
log_group_name = args[0]
filter_name = args[1]
filter_pattern = args[2]
destination_arn = args[3]
role_arn = args[4]
log_group = self.groups.get(log_group_name)
if not log_group:
raise ResourceNotFoundException("The specified log group does not exist.")
if ":lambda:" in destination_arn:
client = aws_stack.connect_to_service("lambda")
lambda_name = aws_stack.lambda_function_name(destination_arn)
try:
client.get_function(FunctionName=lambda_name)
except Exception:
raise InvalidParameterException(
"destinationArn for vendor lambda cannot be used with roleArn"
)
elif ":kinesis:" in destination_arn:
client = aws_stack.connect_to_service("kinesis")
stream_name = aws_stack.kinesis_stream_name(destination_arn)
try:
client.describe_stream(StreamName=stream_name)
except Exception:
raise InvalidParameterException(
"Could not deliver test message to specified Kinesis stream. "
"Check if the given kinesis stream is in ACTIVE state. "
)
elif ":firehose:" in destination_arn:
client = aws_stack.connect_to_service("firehose")
firehose_name = aws_stack.firehose_name(destination_arn)
try:
client.describe_delivery_stream(DeliveryStreamName=firehose_name)
except Exception:
raise InvalidParameterException(
"Could not deliver test message to specified Firehose stream. "
"Check if the given Firehose stream is in ACTIVE state."
)
else:
service = aws_stack.extract_service_from_arn(destination_arn)
raise InvalidParameterException(
f"PutSubscriptionFilter operation cannot work with destinationArn for vendor {service}"
)
if filter_pattern:
for stream in log_group.streams.values():
stream.filter_pattern = filter_pattern
log_group.put_subscription_filter(filter_name, filter_pattern, destination_arn, role_arn)
def put_log_events_model(self, log_group_name, log_stream_name, log_events,
sequence_token):
self.lastIngestionTime = int(unix_time_millis())
self.storedBytes += sum(
[len(log_event['message']) for log_event in log_events])
events = [
logs_models.LogEvent(self.lastIngestionTime, log_event)
for log_event in log_events
]
self.events += events
self.uploadSequenceToken += 1
log_events = [{
'id': event.eventId,
'timestamp': event.timestamp,
'message': event.message,
} for event in events]
data = {
'messageType': 'DATA_MESSAGE',
'owner': aws_stack.get_account_id(),
'logGroup': log_group_name,
'logStream': log_stream_name,
'subscriptionFilters': [self.filter_name],
'logEvents': log_events,
}
output = io.BytesIO()
with GzipFile(fileobj=output, mode='w') as f:
f.write(json.dumps(data, separators=(',', ':')).encode('utf-8'))
payload_gz_encoded = base64.b64encode(
output.getvalue()).decode('utf-8')
event = {'awslogs': {'data': payload_gz_encoded}}
if self.destination_arn:
if ':lambda:' in self.destination_arn:
client = aws_stack.connect_to_service('lambda')
lambda_name = aws_stack.lambda_function_name(
self.destination_arn)
client.invoke(FunctionName=lambda_name,
Payload=json.dumps(event))
if ':kinesis:' in self.destination_arn:
client = aws_stack.connect_to_service('kinesis')
stream_name = aws_stack.kinesis_stream_name(
self.destination_arn)
client.put_record(StreamName=stream_name,
Data=json.dumps(payload_gz_encoded),
PartitionKey=log_group_name)
if ':firehose:' in self.destination_arn:
client = aws_stack.connect_to_service('firehose')
firehose_name = aws_stack.firehose_name(self.destination_arn)
client.put_record(
DeliveryStreamName=firehose_name,
Record={'Data': json.dumps(payload_gz_encoded)})
def get_stream_info(
stream_name,
log_file=None,
shards=None,
env=None,
endpoint_url=None,
ddb_lease_table_suffix=None,
env_vars=None,
):
if env_vars is None:
env_vars = {}
if not ddb_lease_table_suffix:
ddb_lease_table_suffix = DEFAULT_DDB_LEASE_TABLE_SUFFIX
# construct stream info
env = aws_stack.get_environment(env)
props_file = os.path.join(tempfile.gettempdir(),
"kclipy.%s.properties" % short_uid())
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
app_name = "%s%s" % (stream_name, ddb_lease_table_suffix)
stream_info = {
"name": stream_name,
"region": aws_stack.get_region(),
"shards": shards,
"properties_file": props_file,
"log_file": log_file,
"app_name": app_name,
"env_vars": env_vars,
}
# set local connection
if aws_stack.is_local_env(env):
stream_info["conn_kwargs"] = {
"host": LOCALHOST,
"port": config.service_port("kinesis"),
"is_secure": bool(config.USE_SSL),
}
if endpoint_url:
if "conn_kwargs" not in stream_info:
stream_info["conn_kwargs"] = {}
url = urlparse(endpoint_url)
stream_info["conn_kwargs"]["host"] = url.hostname
stream_info["conn_kwargs"]["port"] = url.port
stream_info["conn_kwargs"]["is_secure"] = url.scheme == "https"
return stream_info
def start_kcl_client_process(
stream_name,
listener_script,
log_file=None,
env=None,
configs=None,
endpoint_url=None,
ddb_lease_table_suffix=None,
env_vars=None,
region_name=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL,
log_subscribers=None,
):
if configs is None:
configs = {}
if env_vars is None:
env_vars = {}
if log_subscribers is None:
log_subscribers = []
env = aws_stack.get_environment(env)
# make sure to convert stream ARN to stream name
stream_name = aws_stack.kinesis_stream_name(stream_name)
# decide which credentials provider to use
credentialsProvider = None
if ("AWS_ASSUME_ROLE_ARN" in os.environ or "AWS_ASSUME_ROLE_ARN"
in env_vars) and ("AWS_ASSUME_ROLE_SESSION_NAME" in os.environ
or "AWS_ASSUME_ROLE_SESSION_NAME" in env_vars):
# use special credentials provider that can assume IAM roles and handle temporary STS auth tokens
credentialsProvider = "cloud.localstack.DefaultSTSAssumeRoleSessionCredentialsProvider"
# pass through env variables to child process
for var_name in [
"AWS_ASSUME_ROLE_ARN",
"AWS_ASSUME_ROLE_SESSION_NAME",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
]:
if var_name in os.environ and var_name not in env_vars:
env_vars[var_name] = os.environ[var_name]
if aws_stack.is_local_env(env):
# need to disable CBOR protocol, enforce use of plain JSON,
# see https://github.com/mhart/kinesalite/issues/31
env_vars["AWS_CBOR_DISABLE"] = "true"
if kcl_log_level or (len(log_subscribers) > 0):
if not log_file:
log_file = LOG_FILE_PATTERN.replace("*", short_uid())
TMP_FILES.append(log_file)
run("touch %s" % log_file)
# start log output reader thread which will read the KCL log
# file and print each line to stdout of this process...
reader_thread = OutputReaderThread({
"file": log_file,
"level": kcl_log_level,
"log_prefix": "KCL",
"log_subscribers": log_subscribers,
})
reader_thread.start()
# construct stream info
stream_info = get_stream_info(
stream_name,
log_file,
env=env,
endpoint_url=endpoint_url,
ddb_lease_table_suffix=ddb_lease_table_suffix,
env_vars=env_vars,
)
props_file = stream_info["properties_file"]
# set kcl config options
kwargs = {"metricsLevel": "NONE", "initialPositionInStream": "LATEST"}
# set parameters for local connection
if aws_stack.is_local_env(env):
kwargs[
"kinesisEndpoint"] = f"{LOCALHOST}:{config.service_port('kinesis')}"
kwargs[
"dynamodbEndpoint"] = f"{LOCALHOST}:{config.service_port('dynamodb')}"
kwargs["kinesisProtocol"] = config.get_protocol()
kwargs["dynamodbProtocol"] = config.get_protocol()
kwargs["disableCertChecking"] = "true"
kwargs.update(configs)
# create config file
kclipy_helper.create_config_file(
config_file=props_file,
executableName=listener_script,
streamName=stream_name,
applicationName=stream_info["app_name"],
credentialsProvider=credentialsProvider,
region_name=region_name,
**kwargs,
)
TMP_FILES.append(props_file)
# start stream consumer
stream = KinesisStream(id=stream_name, params=stream_info)
thread_consumer = KinesisProcessorThread.start_consumer(stream)
TMP_THREADS.append(thread_consumer)
return thread_consumer
