class ConfluentKafkaWriter(KafkaWriter):
def __init__(self, kafka_hosts: List[str]):
self.config = {'bootstrap.servers': ','.join(kafka_hosts)}
self.producer = Producer(**self.config)
self.messages: List[Message] = []
def write(self, topic: str, partiton: int, message: Message):
self.producer.produce(topic=topic,
value=message.value,
key=message.key,
partition=partiton,
on_delivery=self.delivery_callback,
timestamp=message.timestamp,
headers=message.headers)
self.producer.poll(0)
def flush(self):
self.producer.flush()
def delivery_callback(self, err, msg):
if err:
logger.error('%% Message failed delivery: %s\n' % err)
else:
self.messages.append(
Message(msg.offset(), msg.key(), msg.value(),
msg.timestamp()[1], None))
def close(self):
self.producer.close()
class ConfluentKafkaProducerUtil(object):
def __init__(self, host):
self.host = host
self.producer = Producer(self.host)
def single_producer(self, topic_name, content):
"""
put single message, flush
:param topic_name:
:param content:
:return:
"""
try:
self.producer.produce(topic_name, content)
self.producer.flush()
return True
except Exception as e:
raise e.message
def producer_message(self, topic_name, content):
"""
producer single message, not flush
:return:
"""
try:
self.producer.produce(topic_name, content)
except Exception as e:
raise e.message
def message_flush(self):
"""
message flush
:return:
"""
try:
self.producer.flush()
except Exception as e:
raise e.message
def close(self):
"""
close producer connection
:return:
"""
self.producer.close()
def batch_producer(self, topic_name, content_list):
"""
batch produce message to kafka
:param topic_name:
:param content_list:
:return:
"""
try:
for tmp_message in content_list:
self.producer.produce(topic_name, tmp_message)
self.producer.flush()
except Exception as e:
raise e.message
class MetricsEventsProducer:
def __init__(self):
self.prepareProducer("ReeferTelemetryProducers")
def prepareProducer(self, groupID):
options = {
'bootstrap.servers':
EventBackboneConfiguration.getBrokerEndPoints(),
'group.id': groupID,
}
if (EventBackboneConfiguration.isSecured()):
options['security.protocol'] = 'SASL_SSL'
# If we are connecting to ES on IBM Cloud, the SASL mechanism is plain
if (EventBackboneConfiguration.getKafkaUser() == 'token'):
options['sasl.mechanisms'] = 'PLAIN'
# If we are connecting to ES on OCP, the SASL mechanism is scram-sha-512
else:
options['sasl.mechanisms'] = 'SCRAM-SHA-512'
options['sasl.username'] = EventBackboneConfiguration.getKafkaUser(
)
options[
'sasl.password'] = EventBackboneConfiguration.getKafkaPassword(
)
if (EventBackboneConfiguration.isEncrypted()):
options[
'ssl.ca.location'] = EventBackboneConfiguration.getKafkaCertificate(
)
print("Kafka options are:")
print(options)
self.producer = Producer(options)
def delivery_report(self, err, msg):
""" Called once for each message produced to indicate delivery result.
Triggered by poll() or flush(). """
if err is not None:
print(
str(datetime.datetime.today()) +
' - Message delivery failed: {}'.format(err))
else:
print(
str(datetime.datetime.today()) +
' - Message delivered to {} [{}]'.format(
msg.topic(), msg.partition()))
def publishEvent(self, eventToSend, keyName):
dataStr = json.dumps(eventToSend)
print(dataStr)
self.producer.produce(
EventBackboneConfiguration.getTelemetryTopicName(),
key=eventToSend[keyName],
value=dataStr.encode('utf-8'),
callback=self.delivery_report)
self.producer.flush()
def close(self):
self.producer.close()
class KafkaTaskQueueSender(TaskQueueSenderInterface):
def __init__(self, config: dict):
# Producer configuration
# See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
# See https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka#prerequisites for SSL issues
self.poll_timeout_sec = config.get("poll_timeout_sec", 100)
self.config, self.topic, err = prepare_kafka_config(config)
if not self.config:
raise Exception(f"No config parsed: {err}")
if not self.topic:
raise Exception("No topic configured: {err}")
# Create Producer instance
self.producer = Producer(self.config)
logger.info(f"Producer will send messages to the topic - '{config['connection_string'].split('=')[-1]}'.")
# Wait until all messages have been delivered
logger.info(f"Waiting for {len(self.producer)} deliveries")
self.producer.flush()
def __del__(self):
if self.producer:
self.producer.close()
def send_event(self, msg: typing.Optional[bytes]):
try:
self.producer.produce(self.topic, msg, on_delivery=self.delivery_callback)
except BufferError as e:
logger.info(
f"Local producer queue is full ({len(self.producer)} messages awaiting delivery): try again. {e}"
)
# self.producer.poll(0)
def put_task(self, task: Task):
task_bytes: typing.Optional[bytes] = None
try:
task_bytes = serialize_task(task)
except Exception as e:
logger.error(f"Error while serizlising task: {e}")
logger.error(f" + Task: {task}")
raise e
if not task_bytes:
raise Exception("Could not serialize task")
self.send_event(task_bytes)
@staticmethod
def delivery_callback(err, msg):
"""This is called only when self.producer.poll(0) will be called.
To test it: call self.producer.poll(0) from 'send_event' func.
"""
if err:
logger.error(f"Message failed delivery: {err} ({msg.topic()} [{msg.partition()}] @ {msg.offset()})")
else:
logger.info(f"Message delivered to {msg.topic()} [{msg.partition()}] @ {msg.offset()}")
class MetricsEventsProducer:
def __init__(self):
self.prepareProducer("ReeferTelemetryProducer")
print("MetricsEventsProducer")
def prepareProducer(self, groupID):
options = {
'bootstrap.servers': KAFKA_BROKERS,
'group.id': groupID,
'delivery.timeout.ms': 15000,
'request.timeout.ms': 15000
}
print("kafka-user: "******"--- This is the configuration for the producer: ---")
logging.info('[KafkaProducer] - {}'.format(options))
logging.info("---------------------------------------------------")
self.producer = Producer(options)
def delivery_report(self, err, msg):
""" Called once for each message produced to indicate delivery result.
Triggered by poll() or flush(). """
if err is not None:
logging.info(
str(datetime.datetime.today()) +
' - Message delivery failed: {}'.format(err))
else:
logging.info(
str(datetime.datetime.today()) +
' - Message delivered to {} [{}]'.format(
msg.topic(), msg.partition()))
def publishEvent(self, eventToSend, keyName):
dataStr = json.dumps(eventToSend)
logging.info("Send " + dataStr + " with key " + keyName + " to " +
TOPIC_NAME)
self.producer.produce(TOPIC_NAME,
key=eventToSend[keyName],
value=dataStr.encode('utf-8'),
callback=self.delivery_report)
self.producer.flush()
# self.producer.poll(5)
def close(self):
self.producer.close()
class MetricsEventsProducer:
def __init__(self):
self.currentRuntime = EventBackboneConfiguration.getCurrentRuntimeEnvironment()
self.brokers = EventBackboneConfiguration.getBrokerEndPoints()
self.apikey = EventBackboneConfiguration.getEndPointAPIKey()
self.topic_name = "containerMetrics"
self.prepareProducer("pythonreefermetricproducers")
def prepareProducer(self,groupID):
options ={
'bootstrap.servers': self.brokers,
'group.id': groupID
}
# We need this test as local kafka does not expect SSL protocol.
if (self.apikey != ''):
options['security.protocol'] = 'SASL_SSL'
options['sasl.mechanisms'] = 'PLAIN'
options['sasl.username'] = '******'
options['sasl.password'] = self.apikey
if (self.currentRuntime == 'ICP'):
options['ssl.ca.location'] = 'es-cert.pem'
print(options)
self.producer = Producer(options)
def delivery_report(self,err, msg):
""" Called once for each message produced to indicate delivery result.
Triggered by poll() or flush(). """
if err is not None:
print('Message delivery failed: {}'.format(err))
else:
print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition()))
def publishEvent(self, eventToSend, keyName):
dataStr = json.dumps(eventToSend)
self.producer.produce("containerMetrics",
key=eventToSend[keyName],
value=dataStr.encode('utf-8'),
callback=self.delivery_report)
self.producer.flush()
def close(self):
self.producer.close()
class ProducerQueue(
): # Provides A Queue Based Interface To Send Messages To The Kafka Broker At The Specified Topic #
def __init__(self, Logger: object, Host: str, TopicName: str,
Queue: object, Partition: int): # Initialization #
# Create Local Vars #
self.Logger = Logger
self.Host = Host
self.TopicName = TopicName
self.Queue = Queue
self.Partition = Partition
# Create Producer Object #
Configuration = {'bootstrap.servers': Host}
self.ProducerObject = Producer(Configuration)
# Register Exit Function #
atexit.register(self.Shutdown, self)
# Launch Queue Handler #
self.QueueHandler()
def QueueHandler(self): # Queue Handler Function #
# Enter Main Loop #
while True:
# Produce Message #
Message = self.Queue.get()
self.ProducerObject.Produce(self.TopicName, Message)
def Shutdown(self): # Destroys Connection Object #
# Shutdown Connection #
self.ProducerObject.close()
self.Logger.Log(
f'Producer Instance Exited At Address {str(hex(id(self)))}')
class KafkaProducer(Greenlet):
def __init__(self, params):
Greenlet.__init__(self)
self.params = params
self.producer = Producer(config)
self.runflag = True
self.index = 0
gevent.sleep()
atexit.register(self.cleanup)
def run(self):
global producer_total, consumer_total, producer_topic
# thread is stopped by setting close event
while self.runflag:
self.index += 1
producer_total += 1
val = makeRecord(self.params)
key = str(self.index % 10)
self.producer.produce(producer_topic,
key=key.encode('utf-8'),
value=val.encode('utf-8'))
socketio.emit('newMessage', {'msg': val}, namespace='/producer')
print("-> producer sent: ", val)
smsg = "EPD Simulator Totals --> Produced: {} Consumed: {}".format(
producer_total, consumer_total)
socketio.emit('newMessage', {'msg': smsg}, namespace='/epd')
# add some random time delay between sent messages
delay = 0.5 + 2 * random.random()
gevent.sleep(delay)
def cleanup(self):
try:
self.producer.flush(timeout=0.5)
self.producer.close(timeout=0.5)
except:
print("-> producer not initialized...")
def main():
print("Confluent Kafka Version: %s - Libversion: %s" %
(version(), libversion()))
topic = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace(
'"', '') + ":" + os.environ["MAPR_STREAMS_TOPIC"].replace('"', '')
print("Producing messages to: %s" % topic)
p = Producer({'bootstrap.servers': ''})
# Listen for messages
running = True
lastflush = 0
while running:
curtime = int(time.time())
curts = time.strftime("%m/%d/%Y %H:%M:%S")
message = {}
message['ts'] = curts
message['field1'] = "This is fun"
message['field2'] = "or is it?"
message_json = json.dumps(message)
print(message_json)
try:
p.produce(topic, value=message_json, callback=delivery_callback)
p.poll(0)
except BufferError as e:
print("Buffer full, waiting for free space on the queue")
p.poll(10)
p.produce(topic, value=message_json, callback=delivery_callback)
except KeyboardInterrupt:
print("\n\nExiting per User Request")
p.close()
sys.exit(0)
delay = random.randint(1, 9)
print("Sleeping for %s" % delay)
time.sleep(delay)
def signal_handler(sig,frame):
producer.close()
sys.exit(0)
def parseArguments():
version="0"
arg2="./data/products.json"
if len(sys.argv) == 1:
print("Usage: SendProductToKafka --file datafilename ")
exit(1)
else:
for idx in range(1, len(sys.argv)):
arg=sys.argv[idx]
if arg == "--file":
arg2=sys.argv[idx+1]
if arg == "--help":
print("Send product json documents to a kafka cluster. Use environment variables KAFKA_BROKERS")
print(" and KAFKA_APIKEY is the cluster accept sasl connection with token user")
print(" and KAFKA_CERT for ca.crt to add for TLS communication")
exit(0)
return arg2
if __name__ == "__main__":
filename = parseArguments()
products = readProducts(filename)
try:
publishEvent(SOURCE_TOPIC,products)
except KeyboardInterrupt:
producer.close()
sys.exit(0)
class KafkaTaskQueueSender(TaskQueueSenderInterface):
# This is not possible in Azure as it is expected you create your topics (a.k.a. event hubs)
# Manually in Azure portal, or using azure proprietary APIs
def _create_topics(self):
# Create Admin instance
self.admin = AdminClient(self.config)
# Create topics
fs = self.admin.create_topics([
NewTopic(topic, num_partitions=3, replication_factor=1)
for topic in [self.topic]
])
# Wait for topic creation to complete
for topic, f in fs.items():
try:
logger.info(f"Waiting for topic '{topic}' to be created")
f.result() # The result itself is None
logger.info(f"Topic '{topic}' created")
except Exception as e:
logger.error(f"Failed to create topic '{topic}': {e}")
def __init__(self, config: dict):
# Producer configuration
# See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
# See https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka#prerequisites for SSL issues
connection_string = str(config.get("connection_string"))
if not connection_string:
raise Exception("No connection_string configured")
self.config = prepare_kafka_config(config)
# Find our topic
parts = parse_event_hub_connection_string(connection_string) or {}
if not parts:
raise Exception(f"No parts found in connection_string")
self.topic = parts.get("entity_path")
if not self.topic:
raise Exception("No topic configured")
# self._create_topics()
# Create Producer instance
self.producer = Producer(self.config)
# Wait until all messages have been delivered
logger.info(f"Waiting for {len(self.producer)} deliveries")
self.producer.flush()
def __del__(self):
if self.producer:
self.producer.close()
def send_event(self, msg: typing.Optional[bytes]):
try:
self.producer.produce(self.topic, msg, callback=delivery_callback)
except BufferError as e:
logger.info(
f"Local producer queue is full ({len(self.producer)} messages awaiting delivery): try again"
)
# self.producer.poll(0)
def put_task(self, task: Task):
task_bytes: typing.Optional[bytes] = None
try:
task_bytes = serialize_task(task)
except Exception as e:
logger.error(f"Error while serizlising task: {e}")
logger.error(f" + Task: {task}")
raise e
if not task_bytes:
raise Exception("Could not serialize task")
self.send_event(task_bytes)
class KafkaClient:
def __init__(self,
producer=False,
consumer=False,
groupId=None,
brokers=[],
topics=[]):
if (producer and consumer) or (not producer and not consumer):
raise Exception(KAFKA.ERROR_BOTH_CLIENT)
self.type = KAFKA.CONSUMER_TYPE if consumer else KAFKA.PRODUCER_TYPE
self.specs = {"groupId": groupId, "brokers": brokers, "topics": topics}
if self.type == KAFKA.CONSUMER_TYPE:
self.__setConsumer()
else:
self.__setProducer()
def __setConsumer(self):
"""This function is used by consumer type client as a default,
and is responsible for setting the consumer client and
subscribing to a topic specified"""
self.client = Consumer({
'bootstrap.servers':
','.join(self.specs["brokers"]),
'group.id':
self.specs['groupId'],
'auto.offset.reset':
'earliest'
})
self.client.subscribe(self.specs["topics"])
return self
def __setProducer(self):
"""This function is used by producer type client as a default,
and is responsible to set the producer client"""
self.client = Producer(
{'bootstrap.servers': ','.join(self.specs["brokers"])})
return self
def __useMessage(self, message, actionable, action):
"""This function triggers a actions passed by parameter if
another verification function passed by parameter meet the
its conditions, these functions take the message consumed"""
if actionable(message):
action(message)
return
def __consume(self, actionable, action):
"""This function starts consuming messages fromm the specified broker
and acts on them with the specified functions"""
stop = False
# self.threads = []
while not stop:
message = self.client.poll(KAFKA.CONSUMER_POLL_TIME)
if message != None:
self.__useMessage(message.value().decode("utf-8"), actionable,
action)
self.client.close()
return
def __produce(self, message):
"""This function produces a message the specified broker"""
self.client.produce(self.specs["topics"][0], message)
self.client.flush()
return
def perform(self, message=None, actionable=None, action=None):
"""This function perform the action of consuming or producing
depending of the type of the client"""
if self.type == KAFKA.PRODUCER_TYPE:
if message != None:
self.__produce(message)
else:
print(KAFKA.ERROR_PERFORM_MISSING_PARAMS)
else:
if actionable != None and action != None:
self.__consume(actionable, action)
else:
print(KAFKA.ERROR_PERFORM_MISSING_PARAMS)
return self
class KafkaConnector(object):
"""Simple wrapper class to configure a simple kafka consumer
and producer pair, so that they can be used to perform simple
filter() and map() operations over the received tweets"""
def __init__(
self,
group_id=None,
consumer_topic='consumer_limbo',
producer_topic='consumer_limbo',
logging_topic='minteressa_stats',
bootstrap_servers='kafka:9092'
):
self.group_id = group_id
self.bootstrap_servers = bootstrap_servers
self.consumer_topic = consumer_topic
self.producer_topic = producer_topic
self.logging_topic = logging_topic
self.consumer = None
self.producer = None
def listen(self):
while True:
msg = self.consumer.poll()
if msg is None:
continue
if msg.error():
# Error or event
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
sys.stderr.write(
'%% %s [%d] reached end at offset %d\n' % (
msg.topic(),
msg.partition(),
msg.offset()
)
)
elif msg.error():
# Error
raise KafkaException(msg.error())
else:
# Proper message
sys.stdout.write(
'%s [partition-%d] at offset %d with key %s:\n' %
(
msg.topic(),
msg.partition(),
msg.offset(),
str(msg.key())
)
)
yield msg
def connect(self):
self.consumer = Consumer({
'bootstrap.servers': self.bootstrap_servers,
'group.id': self.group_id,
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
})
print("subscribing to %s" % self.consumer_topic)
self.consumer.subscribe([
self.consumer_topic
])
print("Subscribed to topic %s " % self.consumer_topic)
self.producer = Producer({
'bootstrap.servers': self.bootstrap_servers,
'group.id': self.group_id
})
def send(self, message, producer_topic=None):
producer_topic = producer_topic \
if producer_topic is not None \
else self.producer_topic
self.producer.produce(
producer_topic,
message
)
# self.producer.flush()
def log(self, message, logging_topic=None):
logging_topic = logging_topic \
if logging_topic is not None \
else self.logging_topic
self.producer.produce(logging_topic, message)
self.producer.flush()
def close(self):
self.consumer.close()
self.producer.close()
class KafkaProducer:
def __init__(self,topic_name,cleanup_policy='delete',num_partitions=1,num_replicas=1):
# Topic configuration that you wanna modify
self.num_partitions = num_partitions
self.num_replicas= num_replicas
self.topic_name = topic_name
self.cleanup_policy = cleanup_policy
# If you chose compact, pleas use lz4, for more details - view README.md
if(self.cleanup_policy == 'compact'):
## Configure Topic for compact
self.topic_configuration = {
"cleanup.policy": 'compact',
"compression.type": "lz4",
"delete.retention.ms": 100,
"file.delete.delay.ms": 10
}
else:
## Configure Topic for compact
self.topic_configuration = {
"cleanup.policy": 'delete'
}
# create a dict with Producer configuration
self.producer_configuration = {
"bootstrap.servers": BROKER_URL,
"client.id": f"{self.topic_name}_group"
}
# create a producer
self.producer = Producer(
self.producer_configuration
)
def create_topic(self):
client = AdminClient({"bootstrap.servers":BROKER_URL})
topic_metadata = client.list_topics(timeout = 5)
if (topic_metadata.topics.get(self.topic_name) is not None):
print("Topic already exist")
pass
else:
print(f" Creating New Topic {self.topic_name}")
futures = client.create_topic([
NewTopic(
topic = self.topic_name,
num_partitions = self.num_partitions,
replication_factor = self.num_replicas,
config = self.topic_configuration
)
])
for topic, future in futures.items():
try:
future.result()
print(f"Topic {self.topic_name} Create Sucessuful!!!")
except Exception as e:
print(f"Failed to create topic {self.topic_name}: {e}")
raise
def close(self):
# Close the Producer, firts flush second close
self.producer.flush()
self.producer.close()
print("Produce Close Succesulf!!")
def get_producer():
producer = Producer({'bootstrap.servers':'127.0.0.1:9092'})
yield producer
producer.close()
