def create_stream(stream_name, delivery_stream_type='DirectPut', delivery_stream_type_configuration=None, s3_destination=None, elasticsearch_destination=None): stream = { 'DeliveryStreamType': delivery_stream_type, 'KinesisStreamSourceConfiguration': delivery_stream_type_configuration, 'HasMoreDestinations': False, 'VersionId': '1', 'CreateTimestamp': time.time(), 'DeliveryStreamARN': firehose_stream_arn(stream_name), 'DeliveryStreamStatus': 'ACTIVE', 'DeliveryStreamName': stream_name, 'Destinations': [] } DELIVERY_STREAMS[stream_name] = stream if elasticsearch_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), elasticsearch_update=elasticsearch_destination) if s3_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), s3_update=s3_destination) if delivery_stream_type == 'KinesisStreamAsSource': kinesis_stream_name = delivery_stream_type_configuration.get( 'KinesisStreamARN').split('/')[1] kinesis_connector.listen_to_kinesis(stream_name=kinesis_stream_name, fh_d_stream=stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix='-firehose') return stream
def create_stream(stream_name, delivery_stream_type='DirectPut', delivery_stream_type_configuration=None, s3_destination=None, elasticsearch_destination=None, tags=None, region_name=None): tags = tags or {} stream = { 'DeliveryStreamType': delivery_stream_type, 'KinesisStreamSourceConfiguration': delivery_stream_type_configuration, 'HasMoreDestinations': False, 'VersionId': '1', 'CreateTimestamp': time.time(), 'DeliveryStreamARN': firehose_stream_arn(stream_name), 'DeliveryStreamStatus': 'ACTIVE', 'DeliveryStreamName': stream_name, 'Destinations': [], 'Tags': tags } DELIVERY_STREAMS[stream_name] = stream if elasticsearch_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), elasticsearch_update=elasticsearch_destination) if s3_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), s3_update=s3_destination) # record event event_publisher.fire_event(event_publisher.EVENT_FIREHOSE_CREATE_STREAM, payload={'n': event_publisher.get_hash(stream_name)}) if delivery_stream_type == 'KinesisStreamAsSource': kinesis_stream_name = delivery_stream_type_configuration.get('KinesisStreamARN').split('/')[1] kinesis_connector.listen_to_kinesis( stream_name=kinesis_stream_name, fh_d_stream=stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix='-firehose', region_name=region_name) return stream
def create_stream(stream_name, s3_destination=None): stream = { 'HasMoreDestinations': False, 'VersionId': '1', 'CreateTimestamp': time.time(), 'DeliveryStreamARN': firehose_stream_arn(stream_name), 'DeliveryStreamStatus': 'ACTIVE', 'DeliveryStreamName': stream_name, 'Destinations': [] } DELIVERY_STREAMS[stream_name] = stream if s3_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), s3_update=s3_destination) return stream
def create_stream(stream_name, s3_destination=None, elasticsearch_destination=None): stream = { 'HasMoreDestinations': False, 'VersionId': '1', 'CreateTimestamp': time.time(), 'DeliveryStreamARN': firehose_stream_arn(stream_name), 'DeliveryStreamStatus': 'ACTIVE', 'DeliveryStreamName': stream_name, 'Destinations': [] } DELIVERY_STREAMS[stream_name] = stream if elasticsearch_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), elasticsearch_update=elasticsearch_destination) if s3_destination: update_destination(stream_name=stream_name, destination_id=short_uid(), s3_update=s3_destination) return stream
def Firehose_get_cfn_attribute(self, attribute_name): if attribute_name == 'Arn': return aws_stack.firehose_stream_arn( self.params.get('DeliveryStreamName')) raise UnformattedGetAttTemplateException()
def create_stream( stream_name: str, delivery_stream_type: str = "DirectPut", delivery_stream_type_configuration: Dict = None, s3_destination: Dict = None, elasticsearch_destination: Dict = None, http_destination: Dict = None, tags: Dict[str, str] = None, ): """Create a firehose stream with destination configurations. In case 'KinesisStreamAsSource' is set, creates a listener to process records from the underlying kinesis stream.""" region = FirehoseBackend.get() tags = tags or {} stream = { "DeliveryStreamType": delivery_stream_type, "KinesisStreamSourceConfiguration": delivery_stream_type_configuration, "HasMoreDestinations": False, "VersionId": "1", "CreateTimestamp": time.time(), "DeliveryStreamARN": firehose_stream_arn(stream_name), "DeliveryStreamStatus": "ACTIVE", "DeliveryStreamName": stream_name, "Destinations": [], "Tags": tags, } region.delivery_streams[stream_name] = stream if elasticsearch_destination: update_destination( stream_name=stream_name, destination_id=short_uid(), elasticsearch_update=elasticsearch_destination, ) if s3_destination: update_destination( stream_name=stream_name, destination_id=short_uid(), s3_update=s3_destination, ) if http_destination: update_destination( stream_name=stream_name, destination_id=short_uid(), http_update=http_destination, ) # record event event_publisher.fire_event( event_publisher.EVENT_FIREHOSE_CREATE_STREAM, payload={"n": event_publisher.get_hash(stream_name)}, ) if delivery_stream_type == "KinesisStreamAsSource": kinesis_stream_name = delivery_stream_type_configuration.get( "KinesisStreamARN").split("/")[1] kinesis_connector.listen_to_kinesis( stream_name=kinesis_stream_name, fh_d_stream=stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix="-firehose", ) return stream
def create_delivery_stream( self, context: RequestContext, delivery_stream_name: DeliveryStreamName, delivery_stream_type: DeliveryStreamType = DeliveryStreamType. DirectPut, kinesis_stream_source_configuration: KinesisStreamSourceConfiguration = None, delivery_stream_encryption_configuration_input: DeliveryStreamEncryptionConfigurationInput = None, s3_destination_configuration: S3DestinationConfiguration = None, extended_s3_destination_configuration: ExtendedS3DestinationConfiguration = None, redshift_destination_configuration: RedshiftDestinationConfiguration = None, elasticsearch_destination_configuration: ElasticsearchDestinationConfiguration = None, amazonopensearchservice_destination_configuration: AmazonopensearchserviceDestinationConfiguration = None, splunk_destination_configuration: SplunkDestinationConfiguration = None, http_endpoint_destination_configuration: HttpEndpointDestinationConfiguration = None, tags: TagDeliveryStreamInputTagList = None, ) -> CreateDeliveryStreamOutput: region = FirehoseBackend.get() destinations: DestinationDescriptionList = [] if elasticsearch_destination_configuration: destinations.append( DestinationDescription( DestinationId=short_uid(), ElasticsearchDestinationDescription= convert_es_config_to_desc( elasticsearch_destination_configuration), )) if amazonopensearchservice_destination_configuration: destinations.append( DestinationDescription( DestinationId=short_uid(), AmazonopensearchserviceDestinationDescription= convert_opensearch_config_to_desc( amazonopensearchservice_destination_configuration), )) if s3_destination_configuration or extended_s3_destination_configuration: destinations.append( DestinationDescription( DestinationId=short_uid(), S3DestinationDescription=convert_s3_config_to_desc( s3_destination_configuration), ExtendedS3DestinationDescription= convert_extended_s3_config_to_desc( extended_s3_destination_configuration), )) if http_endpoint_destination_configuration: destinations.append( DestinationDescription( DestinationId=short_uid(), HttpEndpointDestinationDescription= convert_http_config_to_desc( http_endpoint_destination_configuration), )) if splunk_destination_configuration: LOG.warning( "Delivery stream contains a splunk destination (which is currently not supported)." ) if redshift_destination_configuration: LOG.warning( "Delivery stream contains a redshift destination (which is currently not supported)." ) stream = DeliveryStreamDescription( DeliveryStreamName=delivery_stream_name, DeliveryStreamARN=firehose_stream_arn( stream_name=delivery_stream_name, account_id=context.account_id, region_name=context.region, ), DeliveryStreamStatus=DeliveryStreamStatus.ACTIVE, DeliveryStreamType=delivery_stream_type, HasMoreDestinations=False, VersionId="1", CreateTimestamp=datetime.now(), LastUpdateTimestamp=datetime.now(), Destinations=destinations, Source=convert_source_config_to_desc( kinesis_stream_source_configuration), ) FirehoseBackend.TAGS.tag_resource(stream["DeliveryStreamARN"], tags) region.delivery_streams[delivery_stream_name] = stream # record event event_publisher.fire_event( event_publisher.EVENT_FIREHOSE_CREATE_STREAM, payload={"n": event_publisher.get_hash(delivery_stream_name)}, ) if delivery_stream_type == DeliveryStreamType.KinesisStreamAsSource: if not kinesis_stream_source_configuration: raise InvalidArgumentException( "Missing delivery stream configuration") kinesis_stream_name = kinesis_stream_source_configuration[ "KinesisStreamARN"].split("/")[1] kinesis_connector.listen_to_kinesis( stream_name=kinesis_stream_name, fh_d_stream=delivery_stream_name, listener_func=self._process_records, wait_until_started=True, ddb_lease_table_suffix="-firehose", ) return CreateDeliveryStreamOutput( DeliveryStreamARN=stream["DeliveryStreamARN"])