class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
SCHEMA_REGISTRY_URL = "http://localhost:8081"
BOOTSTRAP_SERVERS = "localhost:9092,localhost:9093,localhost:9094"
admin_client = AdminClient({"bootstrap.servers": BOOTSTRAP_SERVERS})
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
self.broker_properties = {
"bootstrap.servers": Producer.BOOTSTRAP_SERVERS,
"schema.registry.url": Producer.SCHEMA_REGISTRY_URL
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(self.broker_properties)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
Producer.admin_client.create_topics([
NewTopic(self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas),
])
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
Producer.existing_topics.remove(self.topic_name)
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
self.broker_properties = {
"bootstrap.servers": "localhost:9092,localhost:9093,localhost:9094"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(config={
"bootstrap.servers":
"localhost:9092,localhost:9093,localhost:9094",
"schema.registry.url":
"http://localhost:8081",
},
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
admin_client = AdminClient(self.broker_properties)
new_topic = NewTopic(self.topic_name, 1, 1)
admin_client.create_topics([new_topic])
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.producer.flush()
self.producer.close()
logger.info(
f"Producer for topic : {self.topic_name} has been flushed and closed."
)
class Producer:
"""Defines and provides common functionality amongst Producers"""
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
self.broker_properties = {"bootstrap.servers": KAFKA_BROKER_URL}
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
schema_registry = CachedSchemaRegistryClient("http://localhost:8081")
self.producer = AvroProducer(self.broker_properties,
default_key_schema=self.key_schema,
default_value_schema=self.value_schema,
schema_registry=schema_registry)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
client = AdminClient(self.broker_properties)
topic = NewTopic(self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
client.create_topics([topic])
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
Producer.existing_topics = set([])
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
"bootstrap.servers": BROKER_URLS,
"client.id": 1
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(
self.broker_properties,
schema_registry=CachedSchemaRegistryClient(SCHEMA_REGISTRY_URL),
default_key_schema=key_schema,
default_value_schema=value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
client = AdminClient(self.broker_properties)
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
])
for topic, future in futures.items():
try:
future.result()
logger.info(f"topic created: {topic}")
except Exception as e:
logger.info(f"failed to create topic {self.topic_name}: {e}")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.producer.flush(self.timeout_ms)
self.producer.close(self.timeout_ms)
logger.info("producer closed")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# TODO: Configure the broker properties below. Make sure to reference the project README and use the Host URL for Kafka and Schema Registry!
self.broker_properties = {
"bootstrap.servers": BROKER_URL,
"schema.registry.url": url
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
self.producer = AvroProducer(self.broker_properties,
default_key_schema=key_schema,
default_value_schema=value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# TODO: Write code that creates the topic for this producer if it does not already exist on the Kafka Broker.
client = AdminClient({"bootstrap.servers": BROKER_URL})
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
])
for _, future in futures.items():
try:
future.result()
except Exception as e:
pass
logger.info("topic creation kafka integration incomplete - skipping")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.producer.close()
# TODO: Write cleanup code for the Producer here
logger.info("producer close incomplete - skipping")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
self.broker_properties = {
'schema.registry.url': SCHEMA_REGISTRY_URL,
'bootstrap.servers': BROKER_URL,
'client.id': "ex4",
'linger.ms': 1000,
'compression.type': 'lz4',
'batch.num.messages': 100,
}
self.producer = AvroProducer(
self.broker_properties,
default_key_schema=self.key_schema,
default_value_schema=self.value_schema,
)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# If the topic does not already exist, try to create it
if self.topic_name in self.producer.list_topics().topics.keys():
logger.info(
f'Topic {self.topic_name} already exists, skipping topic creation'
)
client = AdminClient({'bootstrap.servers': BROKER_URL})
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas,
config={
"cleanup.policy": "delete",
"compression.type": "lz4",
"delete.retention.ms": 2000,
"file.delete.delay.ms": 2000
})
])
for _, future in futures.items():
try:
future.result()
print(f'Topic: {self.topic_name} successfully created')
except Exception as e:
print(f'Failed to create topic: {self.topic_name}, {e}')
raise
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.producer.flush()
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# Broker Properties
# Documentation for the Producer API is available in
# https://docs.confluent.io/current/clients/python.html
# The avro producer is just wrapper on top of the standard
# confluent python producer. The supported configuration values are dictated
# by the underlying librdkafka, written in C. The documentation of all available
# properties is available in https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
self.broker_properties = {
# Default Kafka configurations
'bootstrap.servers': 'PLAINTEXT://localhost:9092',
'client.id': 'simulation_producer',
# Avro schema
'schema.registry.url': 'http://localhost:8081',
# Batching configurations
'linger.ms': 0.5, # Number of ms to wait to accumulate messages to send
'batch.num.messages': 100, # Number of messages to accumulate before sending
'queue.buffering.max.kbytes': 2097151 # Set to 2GB - default is 16 GB
}
# Configure admin client
self.admin_client = AdminClient(
{'bootstrap.servers': self.broker_properties['bootstrap.servers']}
)
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# Configure the AvroProducer
self.producer = AvroProducer(
config=self.broker_properties
)
# Check if the topic already exists
def topic_exists(self):
logger.info(f"Checking if topic {self.topic_name} already exists")
topic_metadata = self.admin_client.list_topics(timeout=5)
if self.topic_name in topic_metadata.topics:
logger.info(f"Topic {self.topic_name} exists")
return True
else:
return False
return False
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# If topic does not exist, create it
if not self.topic_exists():
# Create new topic object
new_topic = NewTopic(
self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
# Create new topic
self.admin_client.create_topics([new_topic], operation_timeout=10.0)
# Confirm that topic has been created
logger.info(f"Confirming topic {self.topic_name} creation")
# Maybe this is not the best idea to wait here.
# However, better than not having any errors
# Any suggestions on how to handle this ?
time.sleep(0.1)
if not self.topic_exists():
logger.error(f"Could not create topic {self.topic_name}")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
logger.info("Closing producer")
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# * Zookeeper is required every time you need to use a Kafka topics command, it’s specified here because 2hen you call the producer.py you will need to access a topic and thus a zookeeper to manager the broker.
# * Broker list is required anytime you want to interact with a Kafka producer
# * SCHEMA_REGISTRY_URL = A common pattern is to put the instances behind a single virtual IP or round robin DNS such that you can use a single URL in the schema.registry.url configuration but use the entire cluster of Schema Registry instances
# * This points to the schema info that a topic will use
# * It provides a centralise avro schema storage
# * It stores state in a Kafka topic
# Note all upper case naming as these will be set as constances as per PEP8
self.broker_properties = {
"ZOOKEEPER_URL" : "localhost:2181",
"BROKER_URL" : "localhost:9092",
"SCHEMA_REGISTRY_URL" : "http://localhost:8081"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# Configure Avro producer search for exisitng schema & brokers
self.producer = AvroProducer({
'bootstrap.servers': self.broker_properties.get("BROKER_URL"),
'schema.registry.url' : self.broker_properties.get("SCHEMA_REGISTRY_URL")
},
default_key_schema=key_schema,
default_value_schema=value_schema
)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# admin client is needed to access broker info,
adminclient = AdminClient({'bootstrap.servers': self.broker_properties.get("BROKER_URL")})
topics = adminclient.list_topics(timeout=15).topics
newtopic = NewTopic(
self.topic_name,
num_partitions = self.num_partitions,
replication_factor = self.num_replicas
)
futures = adminclient.create_topics([newtopic])
for topic,future in futures.items():
try:
future.result()
logger.debug("topics created")
except:
logger.debug(f"failed to create topic {self.topic_name}")
return
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.producer.flush(timeout = 10)
self.producer.close()
return
logger.info("producer close incomplete - skipping")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
"bootstrap.servers": "PLAINTEXT://localhost:9092",
"schema.registry.url": "http://localhost:8081",
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
self.producer = AvroProducer(
config = self.broker_properties,
default_key_schema = self.key_schema,
default_value_schema = self.value_schema
)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
#
logger.debug("Producer - Creating topic: %s", self.topic_name)
client = AdminClient({'bootstrap.servers': self.broker_properties.get('bootstrap.servers')})
if self.topic_name not in client.list_topics().topics:
futures = client.create_topics([
NewTopic(
topic = self.topic_name,
num_partitions = self.num_partitions,
replication_factor = self.num_replicas,
)
])
for topic, future in futures.items():
try:
future.result()
logger.info("Producer - Topic created: %s", self.topic_name)
except Exception as e:
logger.error("Producer - Failed to create Topic: %s", self.topic_name)
else:
logger.info("Producer - Topic already exists: %s", self.topic_name)
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# TODO: Write cleanup code for the Producer here
#
#
if self.producer is not None:
logger.debug("Producer - Closing Producer")
self.producer.close()
return
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
#self.broker_properties = {
# # TODO
# # TODO
# # TODO
# "kafka" : "PLAINTEXT://localhost:9092",
# "schema_registry" : "http://localhost:8081"
#}
self.broker_properties = {
# TODO
"bootstrap.servers": "PLAINTEXT://localhost:9092",
"schema.registry.url": "http://localhost:8081"
}
# If the topic does not already exist, try to create it
logger.debug("producer - init - self.topic_name:", self.topic_name)
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
# self.producer = AvroProducer(
# )
#schema_registry = CachedSchemaRegistryClient({"url": self.broker_properties["schema_registry"]})
#self.producer = AvroProducer(
# {"bootstrap.servers": self.broker_properties["kafka"]#,
# #"schema.registry.url": self.broker_properties["schema_registry"]
# },
# schema_registry=schema_registry
#)
self.producer = AvroProducer(config=self.broker_properties,
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
logger.debug("producer - initialized")
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
#
#logger.info("topic creation kafka integration incomplete - skipping")
logger.debug("producer - create_topic")
#client = AdminClient({"bootstrap.servers": self.broker_properties["kafka"]})
client = AdminClient(
{"bootstrap.servers": self.broker_properties["bootstrap.servers"]})
logger.debug("producer - create_topic - client created")
futures = client.list_topics()
for topic in futures.topics:
try:
logger.debug(f"list_topics:{topic}")
if topic == self.topic_name:
logger.info(f"{self.topic_name} already exist")
return
except Exception as e:
import traceback
traceback.print_exc()
pass
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
])
for _, future in futures.items():
try:
future.result()
except Exception as e:
import traceback
traceback.print_exc()
pass
# https://knowledge.udacity.com/questions/64633
#if self.producer is not None:
# logger.debug("flushing producer...")
# self.producer.flush()
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# TODO: Write cleanup code for the Producer here
#
#
#logger.info("producer close incomplete - skipping")
if self.topic_name in Producer.existing_topics:
Producer.existing_topics.remove(self.topic_name)
#client = AdminClient({"bootstrap.servers": self.broker_properties["kafka"], 'debug': 'broker,admin' })
client = AdminClient({
"bootstrap.servers":
self.broker_properties["bootstrap.servers"],
'debug':
'broker,admin'
})
futures = client.delete_topics([self.topic_name])
for _, future in futures.items():
try:
future.result()
except Exception as e:
pass
#client.close()
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
self.broker_properties = {
"schema.registry.url": SCHEMA_REGISTRY,
"bootstrap.servers": BOOTSTRAP_SERVER
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(
{
"bootstrap.servers": BOOTSTRAP_SERVER,
"schema.registry.url": SCHEMA_REGISTRY,
},
default_key_schema=key_schema,
default_value_schema=value_schema
)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
admin_client = AdminClient({
"bootstrap.servers": BOOTSTRAP_SERVER
})
if not self.exist_topic(admin_client):
create_topic = admin_client.create_topics([
NewTopic(
topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas,
config={
"cleanup.policy": "delete",
"compression.type": "lz4",
"delete.retention.ms": "100",
"file.delete.delay.ms": "100",
},
)
])
for topic, future in create_topic.items():
try:
future.result()
logger.info(f"Confirmed topic {topic} creation")
except Exception as e:
logger.error(f"failed to create topic {topic}: {e}")
def exist_topic(self, admin_client):
topics_meta = admin_client.list_topics()
if self.topic_name in topics_meta.topics:
logger.info(f"Topic {self.topic_name} exists")
return True
return False
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
logger.info("[close {init}]")
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
"ZOOKEEPER_URL": "localhost:2181",
"SCHEMA_REGISTRY_URL": "http://localhost:8081",
"BORKER_URL": "PLAINTEXT://localhost:9092"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
self.producer = AvroProducer(
{
"bootstrap.servers":
self.broker_properties.get('BROKER_URL'),
"schema.registry.url":
self.broker_properties.get("SCHEMA_REGISTRY_URL")
},
default_key_schema=key_schema,
default_value_schema=value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
#
adminclient = AdminClient(
{"bootstrap.servers": self.broker_properties.get('BROKER_URL')})
topics = adminclient.list_topics(timeout=10).topics
newtopic = NewTopic(self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
futures = adminclient.create_topics([newtopic])
for topic, future in futures.items():
try:
future.result()
logger.debug("topics created")
except:
logger.debug(f"failed to create topic {self.topic_name}")
return
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# TODO: Write cleanup code for the Producer here
#
#
# flush method
# Wait for all messages in the Producer queue to be delivered.
# This is a convenience method that calls poll() until len() is zero or the optional timeout elapses.
self.producer.flush(timeout=10)
self.producer.close()
#logger.info("producer close incomplete - skipping")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
self.broker_properties = {
'bootstrap.servers': 'localhost:9092',
'schema.registry.url': 'http://localhost:8081',
'on_delivery': self.delivery_report
}
# If the topic does not already exist, try to create it
if not self.topic_exists(self.topic_name):
self.create_topic()
self.producer = AvroProducer(self.broker_properties,
default_key_schema=key_schema,
default_value_schema=value_schema)
def delivery_report(err, msg):
if err is not None:
print('Message delivery failed: {}'.format(err))
else:
print('Message delivered to {} [{}]'.format(
msg.topic(), msg.partition()))
def create_topic(self):
logger.info(f'Creating topic {self.topic_name}')
self.topic_exists(self.topic_name)
admin_client = KafkaAdminClient(
bootstrap_servers=self.broker_properties['bootstrap.servers'],
client_id=f'producer{self.topic_name}!')
admin_client.create_topics(new_topics=[
NewTopic(name=self.topic_name,
num_partitions=self.num_replicas,
replication_factor=self.num_replicas)
],
validate_only=False)
logger.info(f'Topic {self.topic_name} created')
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
self.producer.close()
logger.info("producer close complete")
def topic_exists(self, topic_name):
consumer = KafkaConsumer(
bootstrap_servers=self.broker_properties['bootstrap.servers'])
broker_topics = consumer.topics()
return topic_name in broker_topics
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# DONE: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
"schema.registry.url": "http://localhost:8081",
"bootstrap.servers": "PLAINTEXT://localhost:9092"
}
# DONE: Configure the AvroProducer
self.producer = AvroProducer(self.broker_properties)
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# DONE: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
# just producing an empty message is creating the topic
self.producer.produce(topic=self.topic_name, value=None, key=None)
logger.info(f"created topic {self.topic_name}")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# DONE Write cleanup code for the Producer here
#
#
#
self.producer.flush() # wait till all messages are delivered
self.producer.close()
logger.info(f"producer {topic_name} closed")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
# TODO
# TODO
# TODO
"bootstrap.servers":
"PLAINTEXT://localhost:9092,PLAINTEXT://localhost:9093,PLAINTEXT://localhost:9094",
"schema.registry.url": "http://localhost:8081"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
# self.producer = AvroProducer(
# )
self.producer = AvroProducer(self.broker_properties)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
#
client = AdminClient(
{"bootstrap.servers": self.broker_properties['bootstrap.servers']})
try:
client.create_topics([
NewTopic(
topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas,
# config={
# "cleanup.policy": "delete",
# "compression.type": "lz4",
# "delete.retention.ms": "2000",
# "file.delete.delay.ms": "2000",
# },
)
])
logger.info("topic %s created" % self.topic_name)
except:
logger.info(
"topic creation kafka integration incomplete - skipping")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# TODO: Write cleanup code for the Producer here
#
#
try:
self.producer.close()
logger.info("producer %s closed" % self.topic_name)
except:
logger.info("producer close incomplete - skipping")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
"BROKER_URL": "PLAINTEXT://localhost:9092",
"SCHEMA_REGISTRY_URL": "http://localhost:8081"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(
{
'bootstrap.servers':
self.broker_properties["BROKER_URL"],
'schema.registry.url':
self.broker_properties["SCHEMA_REGISTRY_URL"]
},
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
client = AdminClient(
{"bootstrap.servers": self.broker_properties["BROKER_URL"]})
topic_exists = client.list_topics(timeout=5).topics.get(
self.topic_name)
if topic_exists is None:
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas,
config={
"cleanup.policy": "compact",
"compression.type": "lz4",
"delete.retention.ms": 200,
"file.delete.delay.ms": 200
})
])
for topic, future in futures.items():
try:
future.result()
except:
logger.debug(f"failed to create topic {self.topic_name}")
return
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
# logger.info("producer close incomplete - skipping")
if self.producer is not None:
self.producer.flush(timeout=10)
self.producer.close()
return
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
BROKER_URL = "PLAINTEXT://localhost:9092"
SCHEMA_URL = "http://localhost:8081"
broker_properties = {
"bootstrap.servers": BROKER_URL,
"schema.registry.url": SCHEMA_URL
}
self.client = AdminClient({"bootstrap.servers": BROKER_URL})
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
self.producer = AvroProducer(broker_properties,
default_key_schema=key_schema,
default_value_schema=value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
if self.topic_exists() is True:
logger.info(f"topic exists: {self.topic_name}")
else:
newTopics = [
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
]
futures = self.client.create_topics(newTopics)
for topic, future in futures.items():
try:
future.result()
logger.info(f"topic created: {self.topic_name}")
except Exception as e:
logger.info(
f"failed to create topic {self.topic_name}: {e}")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# TODO: Write cleanup code for the Producer here
#
self.producer.close()
logger.info(f"producer closed {self.topic_name}")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
def topic_exists(self):
"""Checks if the given topic exists"""
topic_metadata = self.client.list_topics()
return self.topic_name in set(
t.topic for t in iter(topic_metadata.topics.values()))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# use the Host URL for Kafka and Schema Registry!
self.broker_properties = {
"SCHEMA_REGISTRY_URL": SCHEMA_REGISTRY_URL,
"BROKER_URL": "PLAINTEXT://localhost:9092"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(
{
"bootstrap.servers": BROKER_URL,
"schema.registry.url": SCHEMA_REGISTRY_URL
},
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
logger.info("topic creation kafka integration")
client = AdminClient({"bootstrap.servers": BROKER_URL})
futures = client.create_topics([
NewTopic(
topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_partitions,
config={
"cleanup.policy": "delete",
"compression.type": "lz4",
"delete.retention.ms": "2000",
"file.delete.delay.ms": "2000",
},
)
])
for topic, future in futures.items():
try:
future.result()
logger.info(f"topic {self.topic_name} created")
except Exception as e:
logger.info(f"failed to create topic {self.topic_name}: {e}")
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.producer.close()
logger.info("producer close")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# Done: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
'BROKER_URL': 'localhost:9092',
'SCHEMA_REGISTRY_URL': 'http://localhost:8081'
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# Done: Configure the AvroProducer
schema_registry = CachedSchemaRegistryClient(
self.broker_properties.get('SCHEMA_REGISTRY_URL'))
self.producer = AvroProducer(
{'bootstrap.servers': self.broker_properties.get('BROKER_URL')},
schema_registry=schema_registry,
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
def _getClient(self):
client = AdminClient(
{"bootstrap.servers": self.broker_properties.get('BROKER_URL')})
return client
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# Done: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
#
#
topic_config = {
'cleanup.policy': 'delete',
'compression.type': 'lz4',
'delete.retention.ms': 2000,
'file.delete.delay.ms': 2000
}
topic_check.build_topic(
self.topic_name,
broker_url=self.broker_properties.get('BROKER_URL'),
config_dict=topic_config,
num_partitions=self.num_partitions,
num_replicas=self.num_replicas)
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
#
#
# Done: Write cleanup code for the Producer here
#
#
try:
self.producer.close()
except Exception as e:
self.producer = None
logger.debug(
"close producer occur exception, set producer object to none as substitute: {e}"
)
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# Broker properties
self.broker_properties = {
"ZOOKEEPER_URL": "localhost:2181",
"BROKER_URL": "localhost:9092",
"SCHEMA_REGISTRY_URL": "http://localhost:8081"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
self.producer = AvroProducer(
{
'bootstrap.servers':
self.broker_properties.get("BROKER_URL"),
'schema.registry.url':
self.broker_properties.get("SCHEMA_REGISTRY_URL")
},
default_key_schema=key_schema,
default_value_schema=value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
adminclient = AdminClient(
{'bootstrap.servers': self.broker_properties.get("BROKER_URL")})
topics = adminclient.list_topics(timeout=10).topics
newtopic = NewTopic(self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
futures = adminclient.create_topics([newtopic])
for topic, future in futures.items():
try:
future.result()
logger.debug("topics created")
except:
logger.debug(f"failed to create topic {self.topic_name}")
return
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""
Prepares the producer for exit by cleaning up the producer
"""
# cleanup code for the Producer here
self.producer.flush(timeout=10)
self.producer.close()
return
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
#
#
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
#
#
self.broker_properties = {
"bootstrap.servers": BROKER_URL,
"linger.ms": 1000,
"batch.num.messages": 100,
"compression.type": "lz4"
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
self.producer = AvroProducer({
'bootstrap.servers': BROKER_URL,
'schema.registry.url': SCHEMA_REGISTRY_URL,
'default.topic.config': {'acks': 'all'}},
default_value_schema=value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
#
#
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
client = AdminClient({"bootstrap.servers": BROKER_URL})
futures = client.create_topics([NewTopic(
topic=self.topic_nname,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas
)]
)
for topic, future in futures.items():
try:
future.result()
logger.info(f"topic {topic} was successfully created")
except Exception as e:
print(e)
logger.info(f"topic '{topic}' creation kafka integration incomplete - skipping")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
try:
self.producer.close()
except RuntimeError as re:
logger.info(f"ERROR: {re}\nproducer close incomplete - skipping")
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# Broker properties
self.broker_properties = {
"schema.registry.url": SCHEMA_REGISTRY_URL,
"bootstrap.servers": BROKER_URL
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# Configure the AvroProducer
self.producer = AvroProducer(config=self.broker_properties,
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# Create topic
client = AdminClient({"bootstrap.servers": BROKER_URL})
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
])
for topic, future in futures.items():
try:
future.result()
logger.info(
f"topic creation kafka integration succesfull - topic: {self.topic_name}"
)
except Exception as e:
logger.error(f"{self.topic_name} topic not created: {e}")
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
# Close producer
self.producer.flush()
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
class Producer:
"""Defines and provides common functionality amongst Producers"""
# Tracks existing topics across all Producer instances
existing_topics = set([])
def __init__(
self,
topic_name,
key_schema,
value_schema=None,
num_partitions=1,
num_replicas=1,
):
"""Initializes a Producer object with basic settings"""
self.topic_name = topic_name
self.key_schema = key_schema
self.value_schema = value_schema
self.num_partitions = num_partitions
self.num_replicas = num_replicas
# TODO: Configure the broker properties below. Make sure to reference the project README
# and use the Host URL for Kafka and Schema Registry!
self.broker_properties = {
BROKER_URL_KEY: BROKER_URL,
'schema.registry.url': SCHEMA_REGISTRY_URL,
}
# If the topic does not already exist, try to create it
if self.topic_name not in Producer.existing_topics:
self.create_topic()
Producer.existing_topics.add(self.topic_name)
# TODO: Configure the AvroProducer
self.producer = AvroProducer(self.broker_properties,
default_key_schema=self.key_schema,
default_value_schema=self.value_schema)
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# TODO: Write code that creates the topic for this producer if it does not already exist on
# the Kafka Broker.
client = AdminClient({BROKER_URL_KEY: BROKER_URL})
# check if topic exists, in that case it prints an error message and exits method
if self.topic_exists(client, self.topic_name):
print(f"Topic {self.topic_name} exists")
return
newTopics = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas,
config={
"cleanup.policy": "compact",
"compression.type": "lz4",
"delete.retention.ms": "100",
"file.delete.delay.ms": "100"
})
])
for topic, newTopic in newTopics.items():
try:
newTopic.result()
print(f"topic {self.topic_name} created")
except Exception as e:
print(f"failed to create topic {self.topic_name}: {e}")
raise
def topic_exists(self, client, topicName):
topic_metadata = client.list_topics(timeout=5)
return topic_metadata.topics.get(topicName) is not None
def time_millis(self):
return int(round(time.time() * 1000))
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
# TODO: Write cleanup code for the Producer here
if self.producer is not None:
self.producer.flush()
self.producer.close()
def time_millis(self):
"""Use this function to get the key for Kafka Events"""
return int(round(time.time() * 1000))
