class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts_list, topic, key=None):
logging.Handler.__init__(self)
self.kafka_client = KafkaClient(hosts_list)
self.key = key
self.kafka_topic_name = topic
if not key:
self.producer = SimpleProducer(self.kafka_client)
else:
self.producer = KeyedProducer(self.kafka_client)
def emit(self, record):
# drop kafka logging to avoid infinite recursion
if record.name == 'kafka':
return
try:
# use default formatting
msg = self.format(record)
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send(self.kafka_topic_name, self.key, msg)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self.producer.stop()
logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts_list, topic, key=None):
logging.Handler.__init__(self)
self.kafka_client = KafkaClient(hosts_list)
self.key = key
self.kafka_topic_name = topic
if not key:
self.producer = SimpleProducer(self.kafka_client)
else:
self.producer = KeyedProducer(self.kafka_client)
def emit(self, record):
# drop kafka logging to avoid infinite recursion
if record.name == 'kafka':
return
try:
# use default formatting
msg = self.format(record)
if isinstance(msg, unicode):
msg = msg.encode("utf-8")
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send(self.kafka_topic_name, self.key, msg)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self.producer.stop()
logging.Handler.close(self)
def offsetCommit():
global users
checkUserPartitionMapping()
kafkaClient = KafkaClient(kafkaHost, timeout=None)
producer = KeyedProducer(kafkaClient, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
for partition in partitions:
encodedMessage = simplejson.dumps({'turtleName':turtleName,
'user':'',
'operation':'offsetCommit'})
print producer.send(kafkaTopic, partition, encodedMessage)
producer.stop(1)
kafkaClient.close()
