def connect(broker):
kafka_client = kafka.KafkaClient(broker)
producer = kafka.SimpleProducer(kafka_client)
#producer = kafka.KafkaProducer(bootstrap_servers=broker)
return producer
def _ensure_connection(self):
if self.kafka_client:
return
try:
self.kafka_client = kafka.KafkaClient(self.hostaddrs)
self.producer = kafka.SimpleProducer(self.kafka_client)
except KafkaError as e:
LOG.exception(_LE("Kafka Connection is not available: %s"), e)
self.kafka_client = None
def _ensure_connection(self):
if self._producer:
return
try:
client = kafka.KafkaClient("%s:%s" % (self._host, self._port))
self._producer = kafka.SimpleProducer(client)
except Exception as e:
LOG.exception(_LE("Failed to connect to Kafka service: %s"), e)
raise messaging.DeliveryFailure('Kafka Client is not available, '
'please restart Kafka client')
def run_kafka(args):
client = kafka.KafkaClient(args.server)
producer = kafka.SimpleProducer(client)
# invalid message
producer.send_messages(str('test'), json.dumps({}))
# 3x valid messages
for _ in xrange(3):
producer.send_messages(str('test'), json.dumps(bs))
hub = balog.consumers.ConsumerHub()
hub.scan(sample)
engine = balog.engines.KafkaEngine(hub=hub,
kafka_server=args.server,
group=args.group,
topic=args.topic)
engine.run()
def _create_producer(self):
role = self._config.get("role", "producer_and_consumer")
if "producer" not in role:
return
producer_config = self._config.get("producer_config", {})
partitioner_type = producer_config.get("partitioner_type")
if partitioner_type == self.hash_partitioner:
producer = kafka.KeyedProducer(self._client,
partitioner=kafka.HashedPartitioner)
self._producer_type = "keyed"
elif partitioner_type == self.roundrobin_partitioner:
producer = kafka.KeyedProducer(
self._client, partitioner=kafka.RoundRobinPartitioner)
self._producer_type = "keyed"
else:
producer = kafka.SimpleProducer(self._client)
self._producer_type = "simple"
self._producer = producer
def _get_client(self):
if not self.kafka_client:
self.kafka_client = kafka.KafkaClient(
"%s:%s" % (self.host, self.port))
self.kafka_producer = kafka.SimpleProducer(self.kafka_client)
class StdOutListener(StreamListener):
def on_data(self, data):
producer.send_messages(topic, data.encode('utf-8'))
print("Tweet Sent")
return True
def on_error(self, status):
print(status)
## Set access keys from Twitter Developer app info
with open('/home/n/opt/MindBender_BD/Misc/keys') as keys:
twitter_keys = json.load(keys)
consumer_key = twitter_keys["twitter"]["consumer_key"]
consumer_secret = twitter_keys["twitter"]["consumer_secret"]
access_token = twitter_keys["twitter"]["access_token"]
access_secret = twitter_keys["twitter"]["access_secret"]
## Set authorizations
auth = tw.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tw.API(auth, wait_on_rate_limit=True)
## Kafka stream listener (topic created in terminal)
topic = "tweets"
kafka = kf.KafkaClient("localhost:9099")
producer = kf.SimpleProducer(kafka)
listen = StdOutListener()
stream = tw.Stream(auth, listen)
stream.filter(track="bigdata")
