def _new_producer(producer_name: str, bucket_name: str, cluster_name: str,
**kwargs) -> KafkaProducer:
"""
@param producer_name: The name the producer will identify itself with to the cluster
@param cluster_name: The name of the MSK cluster in AWS
@param kwargs:
@return:
"""
bootstrap_servers = _get_broker_tls_string(bucket_name, cluster_name)
logger.info('Found bootstrap servers: {}'.format(bootstrap_servers))
try:
if bootstrap_servers:
logger.info('Attempting to connect to bootstrap server')
producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
client_id=producer_name,
security_protocol='SSL',
api_version=(0, 10, 0),
**kwargs)
attempts = 0
while not producer.bootstrap_connected(
) and attempts < MAX_CONN_ATTEMPTS:
producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
client_id=producer_name,
security_protocol='SSL',
api_version=(0, 10, 0),
**kwargs)
attempts += 1
sleep(
3
) # wait 3 seconds to try and create a new one. Could be modified to be exponential backoff
if producer.bootstrap_connected():
return producer
else:
logger.exception(
"Unable to connect to boostrap servers after {} attempts".
format(MAX_CONN_ATTEMPTS))
raise Exception(
"Unable to connect to boostrap servers after {} attempts".
format(MAX_CONN_ATTEMPTS))
else:
logger.warning(
"Did not create producer as no bootstrap servers could be found"
)
pass
except Exception as e:
logger.exception(
"Failed to connect to cluster {} with broker string {}.".format(
cluster_name, bootstrap_servers))
raise e
class Producer(threading.Thread):
def __init__(self, topic, value, key):
threading.Thread.__init__(self)
self.stop_event = threading.Event()
self.producer = None
self.url = app.config['KAFKA_URL']
self.topic = topic
self.value = value
self.key = key
daemon = True
def stop(self):
self.stop_event.set()
def run(self):
self.producer = KafkaProducer(
bootstrap_servers=self.url,
key_serializer=str.encode,
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
while not self.stop_event.is_set():
self.send(topic=self.topic, value=self.value, key=self.key)
self.stop()
self.producer.close()
def connected(self):
if not self.producer:
return False
return self.producer.bootstrap_connected()
def send(self, **kwargs):
self.producer.send(**kwargs)
class Producer:
def __init__(self, **kwargs):
self.__producer = None
try:
self.__producer = KafkaProducer(**kwargs)
except NoBrokersAvailable as exc:
print(exc)
print('Please make sure the Kafka brokers are available', kwargs)
def send_message(self, topic, **kwargs):
if not self.__producer:
raise Exception('Producer is not yet initiated')
data = []
if 'data' in kwargs:
data = kwargs.pop('data', [])
if isinstance(data, list) and data:
for chunk in data:
key = chunk.get('key', '')
value = chunk.get('value', {})
self.__producer.send(topic, key=key, value=value, **kwargs)
else:
self.__producer.send(topic, **kwargs)
self.__producer.flush()
def is_connected(self):
return self.__producer and self.__producer.bootstrap_connected()
def lambda_handler(event, context):
producer = KafkaProducer(bootstrap_servers="b-2.kafka.server:9092,b-1.kafka.server:9092")
print(producer.bootstrap_connected())
for record in event['Records']:
#Kinesis data is base64 encoded so decode here
payload=base64.b64decode(record["kinesis"]["data"])
print("Decoded payload: " + str(payload))
producer.send("mytopic",payload)
print(producer.send)
return ("Messages Sent to Kafka Topic")
def produce(events, address):
producer = KafkaProducer(bootstrap_servers=address)
if not producer.bootstrap_connected():
raise Exception("Cannot connect to {}".format(address))
for _ in range(events):
event = LoginEvent()
event.user_name = random_user(4)
key = event.user_name.encode('utf-8')
producer.send('logins', key=key,
value=event.SerializeToString()).get(timeout=60)
producer.flush()
producer.close()
def create_kafka_producer(host='localhost', port=9092):
try:
producer = KafkaProducer(bootstrap_servers='{}:{}'.format(host, port),
value_serializer=lambda x: json.dumps(x).encode('utf-8'))
except NoBrokersAvailable:
print('No broker found at {}:{}'.format(host, port))
raise
if producer.bootstrap_connected():
print('Kafka producer connected!')
return producer
else:
print('Failed to establish connection!')
exit(1)
def connect_kafka_producer(self ):
_producer = None
##return _producer
try:
_producer = KafkaProducer(bootstrap_servers=self.KAFKA_SERVERS, api_version=(0, 10))
if not _producer.bootstrap_connected():
#_producer = None
print('Connecting to Kafka server...')
else:
print('Kafka producer connected succesfully!!')
except Exception as ex:
print('Exception while connecting Kafka')
print(str(ex))
return _producer
def connect_kafka_producer():
global KAFKA_SERVERS
_producer = None
return _producer
### XXX TODO
try:
_producer = KafkaProducer(bootstrap_servers=KAFKA_SERVERS, api_version=(0, 10))
if not _producer.bootstrap_connected():
_producer = None
except Exception as ex:
print('Exception while connecting Kafka')
print(str(ex))
return _producer
def process_data_kafka(path_of_data, broker_address, topic_name, **kwargs):
producer = KafkaProducer(
bootstrap_servers=[broker_address],
value_serializer=lambda x: json.dumps(x).encode("utf-8"),
)
if producer.bootstrap_connected():
logger.info("Connected to kafka!")
with open(path_of_data) as file:
reader = csv.DictReader(file)
for row in reader:
producer.send(topic_name, value=row)
producer.flush()
return "Done"
else:
logger.error("Kafka connection failed!")
return "Failed"
def make_producer(self):
kafka_config = Common.yaml_config("kafka_cluster")
connect_config = {}
connect_config["key_serializer"] = lambda v: ujson.dumps(v).encode(
'utf-8')
connect_config["value_serializer"] = lambda v: ujson.dumps(v).encode(
'utf-8')
connect_config["max_block_ms"] = 15000
if all([
kafka_config["sasl_plain_username"],
kafka_config["sasl_plain_password"]
]):
connect_config.update(kafka_config)
else:
connect_config.update(
bootstrap_servers=kafka_config["bootstrap_servers"])
while True:
producer = KafkaProducer(**connect_config)
if not producer.bootstrap_connected():
logging.debug("will retry connect kafka")
continue
logging.debug(f"connect kafka cluster "
f"{kafka_config['bootstrap_servers']} successful")
return producer
def process_data_kafka(path_of_data, broker_address, topic_name, **kwargs):
producer = KafkaProducer(
bootstrap_servers=[broker_address],
value_serializer=lambda x: dumps(x).encode("utf-8"),
)
if producer.bootstrap_connected():
logger.info("Connected to kafka!")
with open(path_of_data) as file:
reader = csv.DictReader(file)
logger.info("File is here!")
for row in reader:
data = dict(row)
# data["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
producer.send(topic_name, value=data)
producer.flush()
return "Done"
else:
logger.error("Kafka connection failed!")
return "Failed"
class KafkaTestProducer:
def __init__(self):
try:
bootstrap_servers = ['localhost:2888']
self.topic_name = 'kafka_tweets'
self.producer = KafkaProducer(bootstrap_servers=['localhost:2888'])
print(self.producer.bootstrap_connected())
except Exception as ex:
print("Error connecting to kafka.", str(ex))
self.producer.close()
def get_twitter_stream(self, twitter_stream):
for line in twitter_stream:
try:
key = line.get('id')
value = line.get('text')
if key is not None and value is not None:
self.producer.send(self.topic_name, str(key).encode("utf-8"),value.encode("utf-8"))
print("Message sent to kafka consumer")
self.producer.flush()
except Exception as ex:
print("Error sending message to kafka.", str(ex))
self.producer.close()
from time import sleep
from json import dumps
from kafka import KafkaProducer
import server_name
servers = server_name.servers
producer = KafkaProducer(bootstrap_servers=servers,
value_serializer=lambda x: dumps(x).encode('utf-8'))
print(producer.bootstrap_connected())
for e in range(1000):
data = {'number': e}
print(data)
producer.send('numtest', value=data)
print('sent')
sleep(5)
import datetime
from kafka import KafkaProducer
import time
producer = KafkaProducer(bootstrap_servers=['my-kafka:9092'])
if not producer.bootstrap_connected():
print("not connected to bootstrap server")
exit()
def err_callback(error):
print("could not send: " + error)
def success_callback(metdata):
print("send successful")
for _ in range(100):
print("sending...")
producer.send('testing-consumer', bytes(str(datetime.date.today()), 'utf-8')).add_callback(success_callback).add_callback(err_callback)
time.sleep(1)
from kafka import KafkaProducer
KAFKA_SERVER = "35.233.3.112:9092"
producer = KafkaProducer(bootstrap_servers=KAFKA_SERVER)
print("Connexion :", producer.bootstrap_connected())
while True:
message = input("Message : ")
future = producer.send('test-kafka', str.encode(message))
result = future.get(timeout=5)
print(result)
# -*- coding: utf-8 -*-
import tweepy
from tweepy.streaming import json
from kafka import KafkaProducer
"""
KAFKA PRODUCER INIT
"""
producer = KafkaProducer(bootstrap_servers="kafka:9092")
topic_name = "tweets-kafka"
if (producer.bootstrap_connected()):
print("Connected to bootstrap")
producer.send(topic_name, "Hello from tweeter".encode("utf-8"))
"""
TWITTER API AUTHENTICATION
"""
consumer_token = ""
consumer_secret = ""
access_token = ""
access_secret = ""
auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
class KafkaNotifier:
""" Simple Kafka producer.
Can be configured with the 'conf.yml' configuration file in the same directory.
The configuration can be extended to support more features:
https://github.com/dpkp/kafka-python/blob/master/kafka/producer/kafka.py
"""
def __init__(self):
self._bootstrap_servers = None
self._default_topic = None
self.init_from_configuration()
try:
self.producer = KafkaProducer(
bootstrap_servers=[self._bootstrap_servers],
value_serializer=lambda x: json.dumps(x).encode('utf-8'),
retries=5,
)
except NoBrokersAvailable as exception:
self.producer = None
print(
colored(
f'[KAFKA] Could not initialize Kafka producer: {exception}>',
'red'))
def __str__(self):
return f'{self.__class__.__name__}({self._bootstrap_servers}, {self._default_topic})'
def __del__(self):
if self.producer is not None:
self.producer.flush()
self.producer.close()
def init_from_configuration(self):
""" Read conf.yml configuration file to setup the producer. """
kafka_config_file = os.path.join(os.path.dirname(__file__), "conf.yml")
with open(kafka_config_file) as file:
conf_dict = yaml.load(file, Loader=yaml.FullLoader)
try:
self._bootstrap_servers = conf_dict['bootstrap_servers']
self._default_topic = conf_dict['topic']
except KeyError as exception:
print(exception)
def send(self, json_payload, topic=None):
""" Send a JSON formatted event json_payload to a Kafka topic.
Use _default_topic if no topic is given.
"""
topic = topic if topic is not None else self._default_topic
if self.producer is not None and self.producer.bootstrap_connected():
try:
self.producer.send(
topic=topic, value=json_payload).add_callback(
on_send_success).add_errback(on_send_error)
# Print event for debug
print(
f'[KAFKA] PUSHED {json_payload} to kafka topic <{topic}>')
except KafkaTimeoutError as exception:
print(
colored(
f"[KAFKA] Caught KafkaTimeoutError exception: {exception}",
"red"))
else:
# print event for traceability if not sent
print(
colored(
f"""[KAFKA] No push on topic [{topic}] for message:
[{json_payload}]
as NOT connected to bootstrap servers [{self._bootstrap_servers}]""", 'red'))
except Exception as e:
print("Unable to open JSON file ")
print("Exception - ", e)
else:
print("JSON File loaded")
try:
# create the Producer instance
service_producer = KafkaProducer(
bootstrap_servers=kafka_server,
value_serializer=lambda x: json.dumps(x).encode('utf-8'),
retries=3)
print("Producer connected: ",
service_producer.bootstrap_connected())
for i in range(len(json_data)):
data, quality_status = quality_checker(json_data[i + 88])
print("sending ... ", i)
if quality_status == 0:
# means quality check passed. Send the data normally
service_producer.send(kafka_event_topic, value=data)
service_producer.flush()
else:
# some or all quality check failed. Send a log/warning/error/repair channel
print("Quality check failed - ", quality_status)
print("Sending data to the repair channel....")
sleep(0.5)
service_producer.close()
except KeyboardInterrupt:
# Define topic name where the message will publish
# topicName = 'pythonTwitter'
# Initialize producer variable
settings = {
"bootstrap_servers": 'localhost:9092',
# "compression_type":'snappy',
"batch_size": 32 * 1024,
"linger_ms": 20,
"acks": 'all',
"retries": 1000,
"max_in_flight_requests_per_connection": 5
}
producer = KafkaProducer(**settings)
if producer.bootstrap_connected() == False:
raise Exception('You arent connected to the kafka server')
import json
class MyStreamListener(tweepy.StreamListener):
def on_error(self, status_code):
if status_code == 420:
#returning False in on_error disconnects the stream
return False
def on_status(self, status):
key = 'Python'.encode()
information = json.dumps(status._json).encode()
producer.send(topicName, information, key=key)
from kafka import KafkaProducer
import msgpack
import time
import json
bootstrap_servers = 'localhost:9092'
value_serializer = msgpack.dumps
topic = 'electric'
producer = KafkaProducer(bootstrap_servers=bootstrap_servers)
print("Connection to kafka : ", producer.bootstrap_connected())
for i in range(100):
time.sleep(1)
# return_obj = producer.send('electric', b'some_message_bytes')
producer = KafkaProducer(value_serializer=lambda m: json.dumps(m).encode('ascii'))
producer.send(topic, {'key': i * 2})
# producer.send('electric', {'key': 'value'})
def image_processing(cam_index):
# cam id
cam_ID = cam_info.cam_IDs[cam_index]
# Video URL
VIDEO_URL = cam_info.VIDEO_URLs[cam_index]
# Camera Info
cam_name = cam_info.addresses[cam_index]
# lat
lat = cam_info.lats[cam_index]
# lon
lon = cam_info.lons[cam_index]
# facing
facing = cam_info.facings[cam_index]
# connect to kafka server and produce topic
producer = KafkaProducer(
bootstrap_servers=servers,
value_serializer=lambda x: dumps(x).encode('utf-8'))
print(producer.bootstrap_connected())
# current script path
scripty_path = os.path.dirname(os.path.realpath(__file__))
# config yolo model
config_path = scripty_path + '/yolo_model/yolov3.cfg'
weight_path = scripty_path + '/yolo_model/yolov3.weights'
class_path = scripty_path + '/yolo_model/yolov3.txt'
# build net
net = cv2.dnn.readNet(weight_path, config_path)
# build output layer
output_layers = fun.get_output_layers(net)
# define classes
classes = None
with open(class_path, 'r') as f:
classes = [line.strip() for line in f.readlines()]
# pre-define color scheme
COLORS = np.random.uniform(0, 255, size=(len(classes), 3))
# read video
cam = cv2.VideoCapture(VIDEO_URL)
while True:
f, im = cam.read()
# count objects in the frame
if im is not None:
image_o, class_ids = fun.object_identification(
im, classes, net, COLORS)
data = {
'cars':
class_ids.count(2),
'trucks':
class_ids.count(7),
'person':
class_ids.count(0),
'bicycle':
class_ids.count(1),
'motorcycle':
class_ids.count(3),
'bus':
class_ids.count(5),
'vehicles':
class_ids.count(2) + class_ids.count(7) + class_ids.count(3) +
class_ids.count(5),
'cam_ID':
cam_ID,
'cam_name':
cam_name,
'lat':
lat,
'lon':
lon,
'facing':
facing,
'time':
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'region':
500
}
# send data to topic
# only if car detected
if class_ids.count(2) + class_ids.count(7) + class_ids.count(
3) + class_ids.count(5) > 0:
producer.send(cam_ID, value=data)
print(cam_index)
