def _new_producer(producer_name: str, bucket_name: str, cluster_name: str, **kwargs) -> KafkaProducer: """ @param producer_name: The name the producer will identify itself with to the cluster @param cluster_name: The name of the MSK cluster in AWS @param kwargs: @return: """ bootstrap_servers = _get_broker_tls_string(bucket_name, cluster_name) logger.info('Found bootstrap servers: {}'.format(bootstrap_servers)) try: if bootstrap_servers: logger.info('Attempting to connect to bootstrap server') producer = KafkaProducer(bootstrap_servers=bootstrap_servers, client_id=producer_name, security_protocol='SSL', api_version=(0, 10, 0), **kwargs) attempts = 0 while not producer.bootstrap_connected( ) and attempts < MAX_CONN_ATTEMPTS: producer = KafkaProducer(bootstrap_servers=bootstrap_servers, client_id=producer_name, security_protocol='SSL', api_version=(0, 10, 0), **kwargs) attempts += 1 sleep( 3 ) # wait 3 seconds to try and create a new one. Could be modified to be exponential backoff if producer.bootstrap_connected(): return producer else: logger.exception( "Unable to connect to boostrap servers after {} attempts". format(MAX_CONN_ATTEMPTS)) raise Exception( "Unable to connect to boostrap servers after {} attempts". format(MAX_CONN_ATTEMPTS)) else: logger.warning( "Did not create producer as no bootstrap servers could be found" ) pass except Exception as e: logger.exception( "Failed to connect to cluster {} with broker string {}.".format( cluster_name, bootstrap_servers)) raise e
class Producer(threading.Thread): def __init__(self, topic, value, key): threading.Thread.__init__(self) self.stop_event = threading.Event() self.producer = None self.url = app.config['KAFKA_URL'] self.topic = topic self.value = value self.key = key daemon = True def stop(self): self.stop_event.set() def run(self): self.producer = KafkaProducer( bootstrap_servers=self.url, key_serializer=str.encode, value_serializer=lambda v: json.dumps(v).encode('utf-8')) while not self.stop_event.is_set(): self.send(topic=self.topic, value=self.value, key=self.key) self.stop() self.producer.close() def connected(self): if not self.producer: return False return self.producer.bootstrap_connected() def send(self, **kwargs): self.producer.send(**kwargs)
class Producer: def __init__(self, **kwargs): self.__producer = None try: self.__producer = KafkaProducer(**kwargs) except NoBrokersAvailable as exc: print(exc) print('Please make sure the Kafka brokers are available', kwargs) def send_message(self, topic, **kwargs): if not self.__producer: raise Exception('Producer is not yet initiated') data = [] if 'data' in kwargs: data = kwargs.pop('data', []) if isinstance(data, list) and data: for chunk in data: key = chunk.get('key', '') value = chunk.get('value', {}) self.__producer.send(topic, key=key, value=value, **kwargs) else: self.__producer.send(topic, **kwargs) self.__producer.flush() def is_connected(self): return self.__producer and self.__producer.bootstrap_connected()
def lambda_handler(event, context): producer = KafkaProducer(bootstrap_servers="b-2.kafka.server:9092,b-1.kafka.server:9092") print(producer.bootstrap_connected()) for record in event['Records']: #Kinesis data is base64 encoded so decode here payload=base64.b64decode(record["kinesis"]["data"]) print("Decoded payload: " + str(payload)) producer.send("mytopic",payload) print(producer.send) return ("Messages Sent to Kafka Topic")
def produce(events, address): producer = KafkaProducer(bootstrap_servers=address) if not producer.bootstrap_connected(): raise Exception("Cannot connect to {}".format(address)) for _ in range(events): event = LoginEvent() event.user_name = random_user(4) key = event.user_name.encode('utf-8') producer.send('logins', key=key, value=event.SerializeToString()).get(timeout=60) producer.flush() producer.close()
def create_kafka_producer(host='localhost', port=9092): try: producer = KafkaProducer(bootstrap_servers='{}:{}'.format(host, port), value_serializer=lambda x: json.dumps(x).encode('utf-8')) except NoBrokersAvailable: print('No broker found at {}:{}'.format(host, port)) raise if producer.bootstrap_connected(): print('Kafka producer connected!') return producer else: print('Failed to establish connection!') exit(1)
def connect_kafka_producer(self ): _producer = None ##return _producer try: _producer = KafkaProducer(bootstrap_servers=self.KAFKA_SERVERS, api_version=(0, 10)) if not _producer.bootstrap_connected(): #_producer = None print('Connecting to Kafka server...') else: print('Kafka producer connected succesfully!!') except Exception as ex: print('Exception while connecting Kafka') print(str(ex)) return _producer
def connect_kafka_producer(): global KAFKA_SERVERS _producer = None return _producer ### XXX TODO try: _producer = KafkaProducer(bootstrap_servers=KAFKA_SERVERS, api_version=(0, 10)) if not _producer.bootstrap_connected(): _producer = None except Exception as ex: print('Exception while connecting Kafka') print(str(ex)) return _producer
def process_data_kafka(path_of_data, broker_address, topic_name, **kwargs): producer = KafkaProducer( bootstrap_servers=[broker_address], value_serializer=lambda x: json.dumps(x).encode("utf-8"), ) if producer.bootstrap_connected(): logger.info("Connected to kafka!") with open(path_of_data) as file: reader = csv.DictReader(file) for row in reader: producer.send(topic_name, value=row) producer.flush() return "Done" else: logger.error("Kafka connection failed!") return "Failed"
def make_producer(self): kafka_config = Common.yaml_config("kafka_cluster") connect_config = {} connect_config["key_serializer"] = lambda v: ujson.dumps(v).encode( 'utf-8') connect_config["value_serializer"] = lambda v: ujson.dumps(v).encode( 'utf-8') connect_config["max_block_ms"] = 15000 if all([ kafka_config["sasl_plain_username"], kafka_config["sasl_plain_password"] ]): connect_config.update(kafka_config) else: connect_config.update( bootstrap_servers=kafka_config["bootstrap_servers"]) while True: producer = KafkaProducer(**connect_config) if not producer.bootstrap_connected(): logging.debug("will retry connect kafka") continue logging.debug(f"connect kafka cluster " f"{kafka_config['bootstrap_servers']} successful") return producer
def process_data_kafka(path_of_data, broker_address, topic_name, **kwargs): producer = KafkaProducer( bootstrap_servers=[broker_address], value_serializer=lambda x: dumps(x).encode("utf-8"), ) if producer.bootstrap_connected(): logger.info("Connected to kafka!") with open(path_of_data) as file: reader = csv.DictReader(file) logger.info("File is here!") for row in reader: data = dict(row) # data["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") producer.send(topic_name, value=data) producer.flush() return "Done" else: logger.error("Kafka connection failed!") return "Failed"
class KafkaTestProducer: def __init__(self): try: bootstrap_servers = ['localhost:2888'] self.topic_name = 'kafka_tweets' self.producer = KafkaProducer(bootstrap_servers=['localhost:2888']) print(self.producer.bootstrap_connected()) except Exception as ex: print("Error connecting to kafka.", str(ex)) self.producer.close() def get_twitter_stream(self, twitter_stream): for line in twitter_stream: try: key = line.get('id') value = line.get('text') if key is not None and value is not None: self.producer.send(self.topic_name, str(key).encode("utf-8"),value.encode("utf-8")) print("Message sent to kafka consumer") self.producer.flush() except Exception as ex: print("Error sending message to kafka.", str(ex)) self.producer.close()
from time import sleep from json import dumps from kafka import KafkaProducer import server_name servers = server_name.servers producer = KafkaProducer(bootstrap_servers=servers, value_serializer=lambda x: dumps(x).encode('utf-8')) print(producer.bootstrap_connected()) for e in range(1000): data = {'number': e} print(data) producer.send('numtest', value=data) print('sent') sleep(5)
import datetime from kafka import KafkaProducer import time producer = KafkaProducer(bootstrap_servers=['my-kafka:9092']) if not producer.bootstrap_connected(): print("not connected to bootstrap server") exit() def err_callback(error): print("could not send: " + error) def success_callback(metdata): print("send successful") for _ in range(100): print("sending...") producer.send('testing-consumer', bytes(str(datetime.date.today()), 'utf-8')).add_callback(success_callback).add_callback(err_callback) time.sleep(1)
from kafka import KafkaProducer KAFKA_SERVER = "35.233.3.112:9092" producer = KafkaProducer(bootstrap_servers=KAFKA_SERVER) print("Connexion :", producer.bootstrap_connected()) while True: message = input("Message : ") future = producer.send('test-kafka', str.encode(message)) result = future.get(timeout=5) print(result)
# -*- coding: utf-8 -*- import tweepy from tweepy.streaming import json from kafka import KafkaProducer """ KAFKA PRODUCER INIT """ producer = KafkaProducer(bootstrap_servers="kafka:9092") topic_name = "tweets-kafka" if (producer.bootstrap_connected()): print("Connected to bootstrap") producer.send(topic_name, "Hello from tweeter".encode("utf-8")) """ TWITTER API AUTHENTICATION """ consumer_token = "" consumer_secret = "" access_token = "" access_secret = "" auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
class KafkaNotifier: """ Simple Kafka producer. Can be configured with the 'conf.yml' configuration file in the same directory. The configuration can be extended to support more features: https://github.com/dpkp/kafka-python/blob/master/kafka/producer/kafka.py """ def __init__(self): self._bootstrap_servers = None self._default_topic = None self.init_from_configuration() try: self.producer = KafkaProducer( bootstrap_servers=[self._bootstrap_servers], value_serializer=lambda x: json.dumps(x).encode('utf-8'), retries=5, ) except NoBrokersAvailable as exception: self.producer = None print( colored( f'[KAFKA] Could not initialize Kafka producer: {exception}>', 'red')) def __str__(self): return f'{self.__class__.__name__}({self._bootstrap_servers}, {self._default_topic})' def __del__(self): if self.producer is not None: self.producer.flush() self.producer.close() def init_from_configuration(self): """ Read conf.yml configuration file to setup the producer. """ kafka_config_file = os.path.join(os.path.dirname(__file__), "conf.yml") with open(kafka_config_file) as file: conf_dict = yaml.load(file, Loader=yaml.FullLoader) try: self._bootstrap_servers = conf_dict['bootstrap_servers'] self._default_topic = conf_dict['topic'] except KeyError as exception: print(exception) def send(self, json_payload, topic=None): """ Send a JSON formatted event json_payload to a Kafka topic. Use _default_topic if no topic is given. """ topic = topic if topic is not None else self._default_topic if self.producer is not None and self.producer.bootstrap_connected(): try: self.producer.send( topic=topic, value=json_payload).add_callback( on_send_success).add_errback(on_send_error) # Print event for debug print( f'[KAFKA] PUSHED {json_payload} to kafka topic <{topic}>') except KafkaTimeoutError as exception: print( colored( f"[KAFKA] Caught KafkaTimeoutError exception: {exception}", "red")) else: # print event for traceability if not sent print( colored( f"""[KAFKA] No push on topic [{topic}] for message: [{json_payload}] as NOT connected to bootstrap servers [{self._bootstrap_servers}]""", 'red'))
except Exception as e: print("Unable to open JSON file ") print("Exception - ", e) else: print("JSON File loaded") try: # create the Producer instance service_producer = KafkaProducer( bootstrap_servers=kafka_server, value_serializer=lambda x: json.dumps(x).encode('utf-8'), retries=3) print("Producer connected: ", service_producer.bootstrap_connected()) for i in range(len(json_data)): data, quality_status = quality_checker(json_data[i + 88]) print("sending ... ", i) if quality_status == 0: # means quality check passed. Send the data normally service_producer.send(kafka_event_topic, value=data) service_producer.flush() else: # some or all quality check failed. Send a log/warning/error/repair channel print("Quality check failed - ", quality_status) print("Sending data to the repair channel....") sleep(0.5) service_producer.close() except KeyboardInterrupt:
# Define topic name where the message will publish # topicName = 'pythonTwitter' # Initialize producer variable settings = { "bootstrap_servers": 'localhost:9092', # "compression_type":'snappy', "batch_size": 32 * 1024, "linger_ms": 20, "acks": 'all', "retries": 1000, "max_in_flight_requests_per_connection": 5 } producer = KafkaProducer(**settings) if producer.bootstrap_connected() == False: raise Exception('You arent connected to the kafka server') import json class MyStreamListener(tweepy.StreamListener): def on_error(self, status_code): if status_code == 420: #returning False in on_error disconnects the stream return False def on_status(self, status): key = 'Python'.encode() information = json.dumps(status._json).encode() producer.send(topicName, information, key=key)
from kafka import KafkaProducer import msgpack import time import json bootstrap_servers = 'localhost:9092' value_serializer = msgpack.dumps topic = 'electric' producer = KafkaProducer(bootstrap_servers=bootstrap_servers) print("Connection to kafka : ", producer.bootstrap_connected()) for i in range(100): time.sleep(1) # return_obj = producer.send('electric', b'some_message_bytes') producer = KafkaProducer(value_serializer=lambda m: json.dumps(m).encode('ascii')) producer.send(topic, {'key': i * 2}) # producer.send('electric', {'key': 'value'})
def image_processing(cam_index): # cam id cam_ID = cam_info.cam_IDs[cam_index] # Video URL VIDEO_URL = cam_info.VIDEO_URLs[cam_index] # Camera Info cam_name = cam_info.addresses[cam_index] # lat lat = cam_info.lats[cam_index] # lon lon = cam_info.lons[cam_index] # facing facing = cam_info.facings[cam_index] # connect to kafka server and produce topic producer = KafkaProducer( bootstrap_servers=servers, value_serializer=lambda x: dumps(x).encode('utf-8')) print(producer.bootstrap_connected()) # current script path scripty_path = os.path.dirname(os.path.realpath(__file__)) # config yolo model config_path = scripty_path + '/yolo_model/yolov3.cfg' weight_path = scripty_path + '/yolo_model/yolov3.weights' class_path = scripty_path + '/yolo_model/yolov3.txt' # build net net = cv2.dnn.readNet(weight_path, config_path) # build output layer output_layers = fun.get_output_layers(net) # define classes classes = None with open(class_path, 'r') as f: classes = [line.strip() for line in f.readlines()] # pre-define color scheme COLORS = np.random.uniform(0, 255, size=(len(classes), 3)) # read video cam = cv2.VideoCapture(VIDEO_URL) while True: f, im = cam.read() # count objects in the frame if im is not None: image_o, class_ids = fun.object_identification( im, classes, net, COLORS) data = { 'cars': class_ids.count(2), 'trucks': class_ids.count(7), 'person': class_ids.count(0), 'bicycle': class_ids.count(1), 'motorcycle': class_ids.count(3), 'bus': class_ids.count(5), 'vehicles': class_ids.count(2) + class_ids.count(7) + class_ids.count(3) + class_ids.count(5), 'cam_ID': cam_ID, 'cam_name': cam_name, 'lat': lat, 'lon': lon, 'facing': facing, 'time': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'region': 500 } # send data to topic # only if car detected if class_ids.count(2) + class_ids.count(7) + class_ids.count( 3) + class_ids.count(5) > 0: producer.send(cam_ID, value=data) print(cam_index)