def find_destination_config_in_args(api_args): """Return (config_arg, config_name) tuple for destination config. Determines which destination config(s) have been specified. The alternative is to use a bunch of 'if' statements to check each destination configuration. If more than one destination config is specified, than an exception is raised. A logical name for the destination type is returned along with the destination config as it's useful way to compare current and replacement destinations. """ destination_names = DESTINATION_TYPES_TO_NAMES.keys() configs = [] for arg_name, arg_value in api_args.items(): # Ignore arguments that are not destination configs. if "_destination" not in arg_name: continue # If the destination config value is non-null, save it. name = arg_name.split("_destination")[0] if name in destination_names and arg_value: configs.append((DESTINATION_TYPES_TO_NAMES[name], arg_value)) # One and only one destination configuration is allowed. if len(configs) != 1: raise InvalidArgumentException( "Exactly one destination configuration is supported for a Firehose" ) return configs[0]
def update_destination( self, delivery_stream_name, current_delivery_stream_version_id, destination_id, s3_destination_update, extended_s3_destination_update, s3_backup_mode, redshift_destination_update, elasticsearch_destination_update, splunk_destination_update, http_endpoint_destination_update, ): # pylint: disable=unused-argument,too-many-arguments,too-many-locals """Updates specified destination of specified delivery stream.""" (destination_name, destination_config) = find_destination_config_in_args(locals()) delivery_stream = self.delivery_streams.get(delivery_stream_name) if not delivery_stream: raise ResourceNotFoundException( f"Firehose {delivery_stream_name} under accountId " f"{get_account_id()} not found.") if destination_name == "Splunk": warnings.warn( "A Splunk destination delivery stream is not yet implemented") if delivery_stream.version_id != current_delivery_stream_version_id: raise ConcurrentModificationException( f"Cannot update firehose: {delivery_stream_name} since the " f"current version id: {delivery_stream.version_id} and " f"specified version id: {current_delivery_stream_version_id} " f"do not match") destination = {} destination_idx = 0 for destination in delivery_stream.destinations: if destination["destination_id"] == destination_id: break destination_idx += 1 else: raise InvalidArgumentException( "Destination Id {destination_id} not found") # Switching between Amazon ES and other services is not supported. # For an Amazon ES destination, you can only update to another Amazon # ES destination. Same with HTTP. Didn't test Splunk. if (destination_name == "Elasticsearch" and "Elasticsearch" not in destination) or (destination_name == "HttpEndpoint" and "HttpEndpoint" not in destination): raise InvalidArgumentException( f"Changing the destination type to or from {destination_name} " f"is not supported at this time.") # If this is a different type of destination configuration, # the existing configuration is reset first. if destination_name in destination: delivery_stream.destinations[destination_idx][ destination_name].update(destination_config) else: delivery_stream.destinations[destination_idx] = { "destination_id": destination_id, destination_name: destination_config, } # Once S3 is updated to an ExtendedS3 destination, both remain in # the destination. That means when one is updated, the other needs # to be updated as well. The problem is that they don't have the # same fields. if destination_name == "ExtendedS3": delivery_stream.destinations[destination_idx][ "S3"] = create_s3_destination_config(destination_config) elif destination_name == "S3" and "ExtendedS3" in destination: destination["ExtendedS3"] = { k: v for k, v in destination["S3"].items() if k in destination["ExtendedS3"] } # Increment version number and update the timestamp. delivery_stream.version_id = str( int(current_delivery_stream_version_id) + 1) delivery_stream.last_update_timestamp = datetime.now( timezone.utc).isoformat()
def create_delivery_stream( self, region, delivery_stream_name, delivery_stream_type, kinesis_stream_source_configuration, delivery_stream_encryption_configuration_input, s3_destination_configuration, extended_s3_destination_configuration, redshift_destination_configuration, elasticsearch_destination_configuration, splunk_destination_configuration, http_endpoint_destination_configuration, tags, ): # pylint: disable=too-many-arguments,too-many-locals,unused-argument """Create a Kinesis Data Firehose delivery stream.""" (destination_name, destination_config) = find_destination_config_in_args(locals()) if delivery_stream_name in self.delivery_streams: raise ResourceInUseException( f"Firehose {delivery_stream_name} under accountId {get_account_id()} " f"already exists") if len(self.delivery_streams) == DeliveryStream.MAX_STREAMS_PER_REGION: raise LimitExceededException( f"You have already consumed your firehose quota of " f"{DeliveryStream.MAX_STREAMS_PER_REGION} hoses. Firehose " f"names: {list(self.delivery_streams.keys())}") # Rule out situations that are not yet implemented. if delivery_stream_encryption_configuration_input: warnings.warn( "A delivery stream with server-side encryption enabled is not " "yet implemented") if destination_name == "Splunk": warnings.warn( "A Splunk destination delivery stream is not yet implemented") if (kinesis_stream_source_configuration and delivery_stream_type != "KinesisStreamAsSource"): raise InvalidArgumentException( "KinesisSourceStreamConfig is only applicable for " "KinesisStreamAsSource stream type") # Validate the tags before proceeding. errmsg = self.tagger.validate_tags(tags or []) if errmsg: raise ValidationException(errmsg) if tags and len(tags) > MAX_TAGS_PER_DELIVERY_STREAM: raise ValidationException( f"1 validation error detected: Value '{tags}' at 'tags' " f"failed to satisify contstraint: Member must have length " f"less than or equal to {MAX_TAGS_PER_DELIVERY_STREAM}") # Create a DeliveryStream instance that will be stored and indexed # by delivery stream name. This instance will update the state and # create the ARN. delivery_stream = DeliveryStream( region, delivery_stream_name, delivery_stream_type, kinesis_stream_source_configuration, destination_name, destination_config, ) self.tagger.tag_resource(delivery_stream.delivery_stream_arn, tags or []) self.delivery_streams[delivery_stream_name] = delivery_stream return self.delivery_streams[delivery_stream_name].delivery_stream_arn