def connect(broker): kafka_client = kafka.KafkaClient(broker) producer = kafka.SimpleProducer(kafka_client) #producer = kafka.KafkaProducer(bootstrap_servers=broker) return producer
def _ensure_connection(self): if self.kafka_client: return try: self.kafka_client = kafka.KafkaClient(self.hostaddrs) self.producer = kafka.SimpleProducer(self.kafka_client) except KafkaError as e: LOG.exception(_LE("Kafka Connection is not available: %s"), e) self.kafka_client = None
def _ensure_connection(self): if self._producer: return try: client = kafka.KafkaClient("%s:%s" % (self._host, self._port)) self._producer = kafka.SimpleProducer(client) except Exception as e: LOG.exception(_LE("Failed to connect to Kafka service: %s"), e) raise messaging.DeliveryFailure('Kafka Client is not available, ' 'please restart Kafka client')
def run_kafka(args): client = kafka.KafkaClient(args.server) producer = kafka.SimpleProducer(client) # invalid message producer.send_messages(str('test'), json.dumps({})) # 3x valid messages for _ in xrange(3): producer.send_messages(str('test'), json.dumps(bs)) hub = balog.consumers.ConsumerHub() hub.scan(sample) engine = balog.engines.KafkaEngine(hub=hub, kafka_server=args.server, group=args.group, topic=args.topic) engine.run()
def _create_producer(self): role = self._config.get("role", "producer_and_consumer") if "producer" not in role: return producer_config = self._config.get("producer_config", {}) partitioner_type = producer_config.get("partitioner_type") if partitioner_type == self.hash_partitioner: producer = kafka.KeyedProducer(self._client, partitioner=kafka.HashedPartitioner) self._producer_type = "keyed" elif partitioner_type == self.roundrobin_partitioner: producer = kafka.KeyedProducer( self._client, partitioner=kafka.RoundRobinPartitioner) self._producer_type = "keyed" else: producer = kafka.SimpleProducer(self._client) self._producer_type = "simple" self._producer = producer
def _get_client(self): if not self.kafka_client: self.kafka_client = kafka.KafkaClient( "%s:%s" % (self.host, self.port)) self.kafka_producer = kafka.SimpleProducer(self.kafka_client)
class StdOutListener(StreamListener): def on_data(self, data): producer.send_messages(topic, data.encode('utf-8')) print("Tweet Sent") return True def on_error(self, status): print(status) ## Set access keys from Twitter Developer app info with open('/home/n/opt/MindBender_BD/Misc/keys') as keys: twitter_keys = json.load(keys) consumer_key = twitter_keys["twitter"]["consumer_key"] consumer_secret = twitter_keys["twitter"]["consumer_secret"] access_token = twitter_keys["twitter"]["access_token"] access_secret = twitter_keys["twitter"]["access_secret"] ## Set authorizations auth = tw.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) api = tw.API(auth, wait_on_rate_limit=True) ## Kafka stream listener (topic created in terminal) topic = "tweets" kafka = kf.KafkaClient("localhost:9099") producer = kf.SimpleProducer(kafka) listen = StdOutListener() stream = tw.Stream(auth, listen) stream.filter(track="bigdata")