def put_log_events_model(self, log_group_name, log_stream_name, log_events,
sequence_token):
# TODO: call/patch upstream method here, instead of duplicating the code!
self.last_ingestion_time = int(unix_time_millis())
self.stored_bytes += sum(
[len(log_event["message"]) for log_event in log_events])
events = [
logs_models.LogEvent(self.last_ingestion_time, log_event)
for log_event in log_events
]
self.events += events
self.upload_sequence_token += 1
log_events = [{
"id": event.event_id,
"timestamp": event.timestamp,
"message": event.message,
} for event in events]
data = {
"messageType": "DATA_MESSAGE",
"owner": aws_stack.get_account_id(),
"logGroup": log_group_name,
"logStream": log_stream_name,
"subscriptionFilters": [self.filter_name],
"logEvents": log_events,
}
output = io.BytesIO()
with GzipFile(fileobj=output, mode="w") as f:
f.write(json.dumps(data, separators=(",", ":")).encode("utf-8"))
payload_gz_encoded = base64.b64encode(
output.getvalue()).decode("utf-8")
event = {"awslogs": {"data": payload_gz_encoded}}
if self.destination_arn:
if ":lambda:" in self.destination_arn:
client = aws_stack.connect_to_service("lambda")
lambda_name = aws_stack.lambda_function_name(
self.destination_arn)
client.invoke(FunctionName=lambda_name,
Payload=json.dumps(event))
if ":kinesis:" in self.destination_arn:
client = aws_stack.connect_to_service("kinesis")
stream_name = aws_stack.kinesis_stream_name(
self.destination_arn)
client.put_record(
StreamName=stream_name,
Data=json.dumps(payload_gz_encoded),
PartitionKey=log_group_name,
)
if ":firehose:" in self.destination_arn:
client = aws_stack.connect_to_service("firehose")
firehose_name = aws_stack.firehose_name(self.destination_arn)
client.put_record(
DeliveryStreamName=firehose_name,
Record={"Data": json.dumps(payload_gz_encoded)},
)
def put_subscription_filter(*args, **kwargs):
log_group_name = args[0]
filter_name = args[1]
filter_pattern = args[2]
destination_arn = args[3]
role_arn = args[4]
log_group = logs_models.logs_backends[
aws_stack.get_region()].groups.get(log_group_name)
if not log_group:
raise ResourceNotFoundException(
'The specified log group does not exist.')
if ':lambda:' in destination_arn:
client = aws_stack.connect_to_service('lambda')
lambda_name = aws_stack.lambda_function_name(destination_arn)
try:
client.get_function(FunctionName=lambda_name)
except Exception:
raise InvalidParameterException(
'destinationArn for vendor lambda cannot be used with roleArn'
)
elif ':kinesis:' in destination_arn:
client = aws_stack.connect_to_service('kinesis')
stream_name = aws_stack.kinesis_stream_name(destination_arn)
try:
client.describe_stream(StreamName=stream_name)
except Exception:
raise InvalidParameterException(
'Could not deliver test message to specified Kinesis stream. '
'Check if the given kinesis stream is in ACTIVE state. '
)
elif ':firehose:' in destination_arn:
client = aws_stack.connect_to_service('firehose')
firehose_name = aws_stack.firehose_name(destination_arn)
try:
client.describe_delivery_stream(
DeliveryStreamName=firehose_name)
except Exception:
raise InvalidParameterException(
'Could not deliver test message to specified Firehose stream. '
'Check if the given Firehose stream is in ACTIVE state.'
)
else:
service = aws_stack.extract_service_from_arn(destination_arn)
raise InvalidParameterException(
'PutSubscriptionFilter operation cannot work with destinationArn for vendor %s'
% service)
log_group.put_subscription_filter(filter_name, filter_pattern,
destination_arn, role_arn)
def moto_put_subscription_filter(fn, self, *args, **kwargs):
log_group_name = args[0]
filter_name = args[1]
filter_pattern = args[2]
destination_arn = args[3]
role_arn = args[4]
log_group = self.groups.get(log_group_name)
if not log_group:
raise ResourceNotFoundException("The specified log group does not exist.")
if ":lambda:" in destination_arn:
client = aws_stack.connect_to_service("lambda")
lambda_name = aws_stack.lambda_function_name(destination_arn)
try:
client.get_function(FunctionName=lambda_name)
except Exception:
raise InvalidParameterException(
"destinationArn for vendor lambda cannot be used with roleArn"
)
elif ":kinesis:" in destination_arn:
client = aws_stack.connect_to_service("kinesis")
stream_name = aws_stack.kinesis_stream_name(destination_arn)
try:
client.describe_stream(StreamName=stream_name)
except Exception:
raise InvalidParameterException(
"Could not deliver test message to specified Kinesis stream. "
"Check if the given kinesis stream is in ACTIVE state. "
)
elif ":firehose:" in destination_arn:
client = aws_stack.connect_to_service("firehose")
firehose_name = aws_stack.firehose_name(destination_arn)
try:
client.describe_delivery_stream(DeliveryStreamName=firehose_name)
except Exception:
raise InvalidParameterException(
"Could not deliver test message to specified Firehose stream. "
"Check if the given Firehose stream is in ACTIVE state."
)
else:
service = aws_stack.extract_service_from_arn(destination_arn)
raise InvalidParameterException(
f"PutSubscriptionFilter operation cannot work with destinationArn for vendor {service}"
)
if filter_pattern:
for stream in log_group.streams.values():
stream.filter_pattern = filter_pattern
log_group.put_subscription_filter(filter_name, filter_pattern, destination_arn, role_arn)
def put_log_events_model(self, log_group_name, log_stream_name, log_events,
sequence_token):
self.lastIngestionTime = int(unix_time_millis())
self.storedBytes += sum(
[len(log_event['message']) for log_event in log_events])
events = [
logs_models.LogEvent(self.lastIngestionTime, log_event)
for log_event in log_events
]
self.events += events
self.uploadSequenceToken += 1
log_events = [{
'id': event.eventId,
'timestamp': event.timestamp,
'message': event.message,
} for event in events]
data = {
'messageType': 'DATA_MESSAGE',
'owner': aws_stack.get_account_id(),
'logGroup': log_group_name,
'logStream': log_stream_name,
'subscriptionFilters': [self.filter_name],
'logEvents': log_events,
}
output = io.BytesIO()
with GzipFile(fileobj=output, mode='w') as f:
f.write(json.dumps(data, separators=(',', ':')).encode('utf-8'))
payload_gz_encoded = base64.b64encode(
output.getvalue()).decode('utf-8')
event = {'awslogs': {'data': payload_gz_encoded}}
if self.destination_arn:
if ':lambda:' in self.destination_arn:
client = aws_stack.connect_to_service('lambda')
lambda_name = aws_stack.lambda_function_name(
self.destination_arn)
client.invoke(FunctionName=lambda_name,
Payload=json.dumps(event))
if ':kinesis:' in self.destination_arn:
client = aws_stack.connect_to_service('kinesis')
stream_name = aws_stack.kinesis_stream_name(
self.destination_arn)
client.put_record(StreamName=stream_name,
Data=json.dumps(payload_gz_encoded),
PartitionKey=log_group_name)
if ':firehose:' in self.destination_arn:
client = aws_stack.connect_to_service('firehose')
firehose_name = aws_stack.firehose_name(self.destination_arn)
client.put_record(
DeliveryStreamName=firehose_name,
Record={'Data': json.dumps(payload_gz_encoded)})
def send_event_to_firehose(event, arn):
delivery_stream_name = aws_stack.firehose_name(arn)
firehose_client = aws_stack.connect_to_service('firehose')
firehose_client.put_record(DeliveryStreamName=delivery_stream_name,
Record={'Data': to_bytes(json.dumps(event))})
def send_event_to_target(
target_arn: str,
event: Dict,
target_attributes: Dict = None,
asynchronous: bool = True,
target: Dict = {},
):
region = target_arn.split(":")[3]
if ":lambda:" in target_arn:
from localstack.services.awslambda import lambda_api
lambda_api.run_lambda(func_arn=target_arn,
event=event,
context={},
asynchronous=asynchronous)
elif ":sns:" in target_arn:
sns_client = connect_to_service("sns", region_name=region)
sns_client.publish(TopicArn=target_arn, Message=json.dumps(event))
elif ":sqs:" in target_arn:
sqs_client = connect_to_service("sqs", region_name=region)
queue_url = get_sqs_queue_url(target_arn)
msg_group_id = dict_utils.get_safe(target_attributes,
"$.SqsParameters.MessageGroupId")
kwargs = {"MessageGroupId": msg_group_id} if msg_group_id else {}
sqs_client.send_message(QueueUrl=queue_url,
MessageBody=json.dumps(event),
**kwargs)
elif ":states:" in target_arn:
stepfunctions_client = connect_to_service("stepfunctions",
region_name=region)
stepfunctions_client.start_execution(stateMachineArn=target_arn,
input=json.dumps(event))
elif ":firehose:" in target_arn:
delivery_stream_name = firehose_name(target_arn)
firehose_client = connect_to_service("firehose", region_name=region)
firehose_client.put_record(
DeliveryStreamName=delivery_stream_name,
Record={"Data": to_bytes(json.dumps(event))},
)
elif ":events:" in target_arn:
if ":api-destination/" in target_arn or ":destination/" in target_arn:
send_event_to_api_destination(target_arn, event,
target.get("HttpParameters"))
else:
events_client = connect_to_service("events", region_name=region)
eventbus_name = target_arn.split(":")[-1].split("/")[-1]
events_client.put_events(
Entries=[{
"EventBusName": eventbus_name,
"Source": event.get("source"),
"DetailType": event.get("detail-type"),
"Detail": event.get("detail"),
}])
elif ":kinesis:" in target_arn:
partition_key_path = dict_utils.get_safe(
target_attributes,
"$.KinesisParameters.PartitionKeyPath",
default_value="$.id",
)
stream_name = target_arn.split("/")[-1]
partition_key = dict_utils.get_safe(event, partition_key_path,
event["id"])
kinesis_client = connect_to_service("kinesis", region_name=region)
kinesis_client.put_record(
StreamName=stream_name,
Data=to_bytes(json.dumps(event)),
PartitionKey=partition_key,
)
elif ":logs:" in target_arn:
log_group_name = target_arn.split(":")[-1]
logs_client = connect_to_service("logs", region_name=region)
log_stream_name = str(uuid.uuid4())
logs_client.create_log_stream(logGroupName=log_group_name,
logStreamName=log_stream_name)
logs_client.put_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
logEvents=[{
"timestamp": now_utc(millis=True),
"message": json.dumps(event)
}],
)
else:
LOG.warning('Unsupported Events rule target ARN: "%s"', target_arn)
