def test_kafka_producer(self):
"""
Test that HerokuKafakProducer does not create any errors
and is an instance of KafkaProducer (python_kafka)
"""
producer = HerokuKafkaProducer(**kafka_params)
assert isinstance(producer, KafkaProducer)
producer.close()
def sendToKafka(data):
producer = HerokuKafkaProducer(
url=KAFKA_URL, # Url string provided by heroku
ssl_cert=KAFKA_CLIENT_CERT, # Client cert string
ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string
ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string
prefix=KAFKA_PREFIX # Prefix provided by heroku,
#,partitioner="0"
)
"""
The .send method will automatically prefix your topic with the KAFKA_PREFIX
NOTE: If the message doesn't seem to be sending try `producer.flush()` to force send.
"""
producer.send(KAFKA_TOPIC_WRITE, data.encode(), partition=0)
def test_kafka_message_send(self):
"""
Test that no errors are thrown, and it doesn't hang when you send a message
"""
producer = HerokuKafkaProducer(**kafka_params)
producer.send(TOPIC1, b"some message")
producer.flush() # Force event send
producer.close()
def test_kafka_message_send_with_key(self):
"""
Test that no errors are thrown, and it doesn't hang when you send a message
"""
producer = HerokuKafkaProducer(**kafka_params)
result = producer.send(topic=TOPIC1,
key=b"hello",
value=b"some message")
producer.flush() # Force event send
producer.close()
def sendToKafka(data, topic=None):
producer = HerokuKafkaProducer(
url= KAFKA_URL, # Url string provided by heroku
ssl_cert= KAFKA_CLIENT_CERT, # Client cert string
ssl_key= KAFKA_CLIENT_CERT_KEY, # Client cert key string
ssl_ca= KAFKA_TRUSTED_CERT, # Client trusted cert string
prefix= KAFKA_PREFIX# Prefix provided by heroku,
#,partitioner="0"
)
"""
The .send method will automatically prefix your topic with the KAFKA_PREFIX
NOTE: If the message doesn't seem to be sending try `producer.flush()` to force send.
"""
if topic==None:
topic=KAFKA_TOPIC_WRITE
logger.debug("about to send {} to topic {}".format(data, topic))
producer.send(topic, data.encode())
producer.flush()
logger.debug("done")
data = file.write(KAFKA_TRUSTED_CERT)
file.close()
KAFKA_PREFIX = os.getenv('KAFKA_PREFIX', '')
KAKFA_TOPIC = "web-to-kafka"
KAFKA_GROUP_ID = "puller"
KAFKA_COMPLETE_TOPIC = KAFKA_PREFIX + KAKFA_TOPIC
"""
All the variable names here match the heroku env variable names.
Just pass the env values straight in and it will work.
"""
producer = HerokuKafkaProducer(
url=KAFKA_URL, # Url string provided by heroku
ssl_cert=KAFKA_CLIENT_CERT, # Client cert string
ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string
ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string
prefix=KAFKA_PREFIX # Prefix provided by heroku,
#,partitioner="0"
)
consumer = HerokuKafkaConsumer(
#KAKFA_TOPIC, # Optional: You don't need to pass any topic at all
url=KAFKA_URL, # Url string provided by heroku
ssl_cert=KAFKA_CLIENT_CERT, # Client cert string
ssl_key=KAFKA_CLIENT_CERT_KEY, # Client cert key string
ssl_ca=KAFKA_TRUSTED_CERT, # Client trusted cert string
prefix=KAFKA_PREFIX, # Prefix provided by heroku,
auto_offset_reset="smallest",
max_poll_records=500,
enable_auto_commit=True,
auto_commit_interval_ms=10,