from film_record import FilmRecord
from logger import CustomLogger
# additional modules for better UX
from utils import Gauge
# Global presets
file_name = "movies.json"
kafka_brockers = ["10.40.1.142:9092", "10.40.1.141:9092"]
kafka_topic_name = "avikulin_test"
kafka_clientid = "Python test util"
kafka_value_serializer = FilmRecord.serialize
# ! Executed code
if __name__ == "__main__":
logger_instance = CustomLogger("kafka_wtire_util")
logger_instance.activate()
films_store = list()
logger_instance.get.info(f"Start reading data from file {file_name}.")
with open(file_name, mode="r") as source_file:
data_store = json.load(source_file)
print("JSON loaded.")
for i, item in enumerate(data_store):
films_store.append(FilmRecord.decode(item))
print(
f"Statistics: count ={len(data_store)}, collection type = {type(data_store)}"
)
print(
f"Film store: count = {len(films_store)}, item type = {type(films_store[0]) if len(films_store) > 0 else 'None'}"
from kafka.structs import TopicPartition, KafkaMessage
from logger import CustomLogger, LogLevels
from sys import exit
# Global presets
kafka_brokers = ["10.40.1.142:9092", "10.40.1.141:9092"]
kafka_topic_name = "avikulin_test"
kafka_consumer_group_id = "test_group#111"
kafka_client_id = __file__
if __name__ == "__main__":
# Enable logging for kafka consumer
kafka_logger = CustomLogger("kafka", log_level= LogLevels.DEBUG)
kafka_logger.activate()
# Enable logging for app
app_logger = CustomLogger("kafka_read_util", log_level=LogLevels.INFO)
app_logger.activate()
while True:
consumer = KafkaConsumer(
kafka_topic_name,
group_id=kafka_consumer_group_id,
client_id=kafka_client_id,
bootstrap_servers=kafka_brokers,
request_timeout_ms=6001,
session_timeout_ms=6000,
heartbeat_interval_ms=2000,
auto_offset_reset="earliest",