def list_tags_for_delivery_stream(self, delivery_stream_name,
exclusive_start_tag_key, limit):
"""Return list of tags."""
result = {"Tags": [], "HasMoreTags": False}
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under account {get_account_id()} "
f"not found.")
tags = self.tagger.list_tags_for_resource(
delivery_stream.delivery_stream_arn)["Tags"]
keys = self.tagger.extract_tag_names(tags)
# If a starting tag is given and can be found, find the index into
# tags, then add one to get the tag following it.
start = 0
if exclusive_start_tag_key:
if exclusive_start_tag_key in keys:
start = keys.index(exclusive_start_tag_key) + 1
limit = limit or MAX_TAGS_PER_DELIVERY_STREAM
result["Tags"] = tags[start:start + limit]
if len(tags) > (start + limit):
result["HasMoreTags"] = True
return result
def untag_delivery_stream(self, delivery_stream_name, tag_keys):
"""Removes tags from specified delivery stream."""
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under account {get_account_id()} "
f"not found.")
# If a tag key doesn't exist for the stream, boto3 ignores it.
self.tagger.untag_resource_using_names(
delivery_stream.delivery_stream_arn, tag_keys)
def describe_delivery_stream(
self,
delivery_stream_name,
limit,
exclusive_start_destination_id,
): # pylint: disable=unused-argument
"""Return description of specified delivery stream and its status.
Note: the 'limit' and 'exclusive_start_destination_id' parameters
are not currently processed/implemented.
"""
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under account {ACCOUNT_ID} "
f"not found.")
result = {"DeliveryStreamDescription": {"HasMoreDestinations": False}}
for attribute, attribute_value in vars(delivery_stream).items():
if not attribute_value:
continue
# Convert from attribute's snake case to camel case for outgoing
# JSON.
name = "".join([x.capitalize() for x in attribute.split("_")])
# Fooey ... always an exception to the rule:
if name == "DeliveryStreamArn":
name = "DeliveryStreamARN"
if name != "Destinations":
if name == "Source":
result["DeliveryStreamDescription"][name] = {
"KinesisStreamSourceDescription": attribute_value
}
else:
result["DeliveryStreamDescription"][name] = attribute_value
continue
result["DeliveryStreamDescription"]["Destinations"] = []
for destination in attribute_value:
description = {}
for key, value in destination.items():
if key == "destination_id":
description["DestinationId"] = value
else:
description[f"{key}DestinationDescription"] = value
result["DeliveryStreamDescription"]["Destinations"].append(
description)
return result
def put_record_batch(self, delivery_stream_name, records):
"""Write multiple data records into a Kinesis Data firehose stream."""
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under account {ACCOUNT_ID} "
f"not found."
)
request_responses = []
for destination in delivery_stream.destinations:
if "ExtendedS3" in destination:
# ExtendedS3 will be handled like S3,but in the future
# this will probably need to be revisited. This destination
# must be listed before S3 otherwise both destinations will
# be processed instead of just ExtendedS3.
request_responses = self.put_s3_records(
delivery_stream_name,
delivery_stream.version_id,
destination["ExtendedS3"],
records,
)
elif "S3" in destination:
request_responses = self.put_s3_records(
delivery_stream_name,
delivery_stream.version_id,
destination["S3"],
records,
)
elif "HttpEndpoint" in destination:
request_responses = self.put_http_records(
destination["HttpEndpoint"], records
)
elif "Elasticsearch" in destination or "Redshift" in destination:
# This isn't implmented as these services aren't implemented,
# so ignore the data, but return a "proper" response.
request_responses = [
{"RecordId": str(uuid4())} for _ in range(len(records))
]
return {
"FailedPutCount": 0,
"Encrypted": False,
"RequestResponses": request_responses,
}
def tag_delivery_stream(self, delivery_stream_name, tags):
"""Add/update tags for specified delivery stream."""
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under account {get_account_id()} "
f"not found.")
if len(tags) > MAX_TAGS_PER_DELIVERY_STREAM:
raise ValidationException(
f"1 validation error detected: Value '{tags}' at 'tags' "
f"failed to satisify contstraint: Member must have length "
f"less than or equal to {MAX_TAGS_PER_DELIVERY_STREAM}")
errmsg = self.tagger.validate_tags(tags)
if errmsg:
raise ValidationException(errmsg)
self.tagger.tag_resource(delivery_stream.delivery_stream_arn, tags)
def delete_delivery_stream(self,
delivery_stream_name,
allow_force_delete=False): # pylint: disable=unused-argument
"""Delete a delivery stream and its data.
AllowForceDelete option is ignored as we only superficially
apply state.
"""
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under account {get_account_id()} "
f"not found.")
self.tagger.delete_all_tags_for_resource(
delivery_stream.delivery_stream_arn)
delivery_stream.delivery_stream_status = "DELETING"
self.delivery_streams.pop(delivery_stream_name)
def update_destination(
self,
delivery_stream_name,
current_delivery_stream_version_id,
destination_id,
s3_destination_update,
extended_s3_destination_update,
s3_backup_mode,
redshift_destination_update,
elasticsearch_destination_update,
splunk_destination_update,
http_endpoint_destination_update,
): # pylint: disable=unused-argument,too-many-arguments,too-many-locals
"""Updates specified destination of specified delivery stream."""
(destination_name,
destination_config) = find_destination_config_in_args(locals())
delivery_stream = self.delivery_streams.get(delivery_stream_name)
if not delivery_stream:
raise ResourceNotFoundException(
f"Firehose {delivery_stream_name} under accountId "
f"{get_account_id()} not found.")
if destination_name == "Splunk":
warnings.warn(
"A Splunk destination delivery stream is not yet implemented")
if delivery_stream.version_id != current_delivery_stream_version_id:
raise ConcurrentModificationException(
f"Cannot update firehose: {delivery_stream_name} since the "
f"current version id: {delivery_stream.version_id} and "
f"specified version id: {current_delivery_stream_version_id} "
f"do not match")
destination = {}
destination_idx = 0
for destination in delivery_stream.destinations:
if destination["destination_id"] == destination_id:
break
destination_idx += 1
else:
raise InvalidArgumentException(
"Destination Id {destination_id} not found")
# Switching between Amazon ES and other services is not supported.
# For an Amazon ES destination, you can only update to another Amazon
# ES destination. Same with HTTP. Didn't test Splunk.
if (destination_name == "Elasticsearch" and "Elasticsearch"
not in destination) or (destination_name == "HttpEndpoint"
and "HttpEndpoint" not in destination):
raise InvalidArgumentException(
f"Changing the destination type to or from {destination_name} "
f"is not supported at this time.")
# If this is a different type of destination configuration,
# the existing configuration is reset first.
if destination_name in destination:
delivery_stream.destinations[destination_idx][
destination_name].update(destination_config)
else:
delivery_stream.destinations[destination_idx] = {
"destination_id": destination_id,
destination_name: destination_config,
}
# Once S3 is updated to an ExtendedS3 destination, both remain in
# the destination. That means when one is updated, the other needs
# to be updated as well. The problem is that they don't have the
# same fields.
if destination_name == "ExtendedS3":
delivery_stream.destinations[destination_idx][
"S3"] = create_s3_destination_config(destination_config)
elif destination_name == "S3" and "ExtendedS3" in destination:
destination["ExtendedS3"] = {
k: v
for k, v in destination["S3"].items()
if k in destination["ExtendedS3"]
}
# Increment version number and update the timestamp.
delivery_stream.version_id = str(
int(current_delivery_stream_version_id) + 1)
delivery_stream.last_update_timestamp = datetime.now(
timezone.utc).isoformat()
