import kafka_utilities
LOG_SENDER = "SQL_DATABASE"
connection_string = """
dbname='{}' host='{}'
user='******' password='******'
""".format(config.SQL_DATABASE, config.SQL_HOST,
config.SQL_USER_NAME, config.SQL_USER_PASSWORD)
try:
connection = db.connect(connection_string)
except Exception as e:
message = "Something went wrong\nwhen connecting to the {} DB: {}".format(
config.SQL_DATABASE, e)
print(message)
kafka_utilities.log(message, LOG_SENDER)
exit(1)
else:
message = "Successfully connected to the {} DB!".format(
config.SQL_DATABASE)
print(message)
kafka_utilities.log(message, LOG_SENDER)
cursor = connection.cursor()
#put a message in DB
def put_record(message):
#get title and the body of a poem
rows = message.splitlines()
def on_message_error(exception):
error_result = "ERROR: {}".format(exception)
print(error_result)
kafka_utilities.log(error_result, LOG_SENDER)
from kafka import KafkaProducer
import json
import config
import kafka_utilities
LOG_SENDER = "PRODUCER"
#creating kafka producer
try:
producer = KafkaProducer(bootstrap_servers = config.KAFKA_BROKERS)
except Exception as e:
message = "ERROR: Something went wrong with initializing the producer: {}".format(e)
print(message)
kafka_utilities.log(message, LOG_SENDER)
exit(1)
else:
kafka_utilities.log("Succesfully initalized the producer", LOG_SENDER)
#success callback
def on_message_success(record_metadata):
message_result = "SUCCESS : message delivered on topic '{}' in partition {} at offset {}".format(
record_metadata.topic, record_metadata.partition, record_metadata.offset)
print(message_result)
kafka_utilities.log(message_result, LOG_SENDER)
#failure callback
def on_message_error(exception):
error_result = "ERROR: {}".format(exception)
print(error_result)
kafka_utilities.log(error_result, LOG_SENDER)
def on_message_success(record_metadata):
message_result = "SUCCESS : message delivered on topic '{}' in partition {} at offset {}".format(
record_metadata.topic, record_metadata.partition, record_metadata.offset)
print(message_result)
kafka_utilities.log(message_result, LOG_SENDER)
import kafka_utilities
import json
import time
import config
LOG_SENDER = "CONSUMER"
#initializing the consumer
try:
consumer = KafkaConsumer(group_id=config.KAFKA_CONSUMER_GROUP_ID,
bootstrap_servers=config.KAFKA_BROKERS)
except Exception as e:
message = "ERROR: Something went wrong with initializing the consumer: {}".format(
e)
print(message)
kafka_utilities.log(message, LOG_SENDER)
exit(1)
else:
kafka_utilities.log("Succesfully initalized the consumer", LOG_SENDER)
#assign partitions mannually in order to read only the latest logs (it's all about the freshest rhymes :) )
available_partitions = consumer.partitions_for_topic(config.KAFKA_TOPIC)
consumer.assign([
TopicPartition(config.KAFKA_TOPIC, partition_id)
for partition_id in available_partitions
])
consumer.seek_to_end()
for message in consumer:
#store a message in SQL db