class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts_list, topic, key=None):
logging.Handler.__init__(self)
self.kafka_client = KafkaClient(hosts_list)
self.key = key
self.kafka_topic_name = topic
if not key:
self.producer = SimpleProducer(self.kafka_client)
else:
self.producer = KeyedProducer(self.kafka_client)
def emit(self, record):
# drop kafka logging to avoid infinite recursion
if record.name == 'kafka':
return
try:
# use default formatting
msg = self.format(record)
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send(self.kafka_topic_name, self.key, msg)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self.producer.stop()
logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts="", topic="", partition=0):
logging.Handler.__init__(self)
self.kafkaClient = KafkaClient(hosts)
self.topic = topic
self.partition = partition
self.producer = KeyedProducer(
self.kafkaClient, async=False, req_acks=KeyedProducer.ACK_AFTER_LOCAL_WRITE, ack_timeout=200
)
def emit(self, record):
# drop kafka logging to avoid infinite recursion
if record.name == "kafka":
return
try:
# use default formatting
msg = self.format(record)
# produce message
self.producer.send_messages(self.topic + record.name, self.partition, msg)
except:
import traceback
ei = sys.exc_info()
traceback.print_exception(ei[0], ei[1], ei[2], None, sys.stderr)
del ei
def close(self):
self.producer.stop()
logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts_list, topic, key=None):
logging.Handler.__init__(self)
self.kafka_client = KafkaClient(hosts_list)
self.key = key
self.kafka_topic_name = topic
if not key:
self.producer = SimpleProducer(self.kafka_client)
else:
self.producer = KeyedProducer(self.kafka_client)
def emit(self, record):
# drop kafka logging to avoid infinite recursion
if record.name == 'kafka':
return
try:
# use default formatting
msg = self.format(record)
if isinstance(msg, unicode):
msg = msg.encode("utf-8")
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send(self.kafka_topic_name, self.key, msg)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self.producer.stop()
logging.Handler.close(self)
def offsetCommit():
global users
checkUserPartitionMapping()
kafkaClient = KafkaClient(kafkaHost, timeout=None)
producer = KeyedProducer(kafkaClient, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
for partition in partitions:
encodedMessage = simplejson.dumps({'turtleName':turtleName,
'user':'',
'operation':'offsetCommit'})
print producer.send(kafkaTopic, partition, encodedMessage)
producer.stop(1)
kafkaClient.close()
class KafkaLoggingHandler(logging.Handler):
"""
Use kafka to send msg to elk platform
"""
def __init__(self, hosts_list, topic,
timeout_secs=DEFAULT_SOCKET_TIMEOUT_SECONDS, **kwargs):
logging.Handler.__init__(self)
self.kafka_client = KafkaClient(hosts_list, timeout=timeout_secs)
self.key = kwargs.get("key", None)
self.kafka_topic_name = topic
if not self.key:
self.producer = SimpleProducer(self.kafka_client, **kwargs)
else:
self.producer = KeyedProducer(self.kafka_client, **kwargs)
def emit(self, record):
"""
emit record
:param record:
:return:
"""
# drop kafka logging to avoid infinite recursion
if record.name == 'kafka':
return
try:
# use default formatting
msg = self.format(record)
msg = msg.encode("utf-8")
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send_messages(
self.kafka_topic_name, self.key, msg)
except (KeyboardInterrupt, SystemExit):
raise
except BaseException:
self.handleError(record)
def close(self):
"""
close the client
:return:
"""
if self.producer is not None:
self.producer.stop()
logging.Handler.close(self)
class Producer(object):
def __init__(self, addr):
self.client = KafkaClient(addr)
self.producer = KeyedProducer(self.client)
def open_save(self, fileName):
log_file = open(fileName, "w")
log_file.close()
return log_file
def create_topic(self, topic):
script = "/usr/local/kafka/bin/kafka-topics.sh"
os.system("{} --create --zookeeper localhost:2181 --topic {} --partitions {} --replication-factor 2".format(script, topic, "4"))
return "topic {} created".format(topic)
def produce_msgs(self, source_symbol, topic):
server_topics = self.client.topic_partitions
if topic not in server_topics:
self.create_topic(topic)
price_field = random.randint(800,1400)
cities = ["Barcelona", "Philadelphia", "Honolulu",
"Atlanta", "Miami", "Chicago", "SF", "LA", "NYC",
"Houston", "Paris", "London", "Tokyo"]
msg_cnt = 0
log_file = open("input1/{}.csv".format(topic), "a")
while True:
time_field = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')
location_field = random.choice(cities)
price_field += random.randint(-10, 10)/10.0
str_fmt = "{},{},{},{}"
message_info = str_fmt.format(source_symbol,
time_field,
location_field,
price_field)
print message_info
log_file.write("{}\n".format(message_info))
self.producer.send_messages(topic, source_symbol, message_info)
msg_cnt += 1
if msg_cnt > 200000:
log_file.close()
self.producer.stop()
break
class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts_list, topic, **kwargs):
logging.Handler.__init__(self)
self.kafka_client = SimpleClient(hosts_list)
self.key = kwargs.get("key", None)
self.kafka_topic_name = topic
if not self.key:
self.producer = SimpleProducer(self.kafka_client, **kwargs)
else:
self.producer = KeyedProducer(self.kafka_client, **kwargs)
def emit(self, record):
# drop kafka logging to avoid infinite recursion
if record.name == 'kafka':
return
try:
# use default formatting
msg = self.format(record)
if isinstance(msg, unicode):
msg = msg.encode("utf-8")
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send_messages(self.kafka_topic_name, self.key,
msg)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
self.handleError(record)
def close(self):
if self.producer is not None:
self.producer.stop()
logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler):
def __init__(self, hosts_list, topic, timeout_secs=DEFAULT_SOCKET_TIMEOUT_SECONDS, **kwargs):
logging.Handler.__init__(self)
self.kafka_client = KafkaClient(hosts_list, timeout=timeout_secs)
self.key = kwargs.get("key", None)
self.kafka_topic_name = topic
if not self.key:
self.producer = SimpleProducer(self.kafka_client, **kwargs)
else:
self.producer = KeyedProducer(self.kafka_client, **kwargs)
self.addFilter(KafkaLoggingFilter())
def emit(self, record):
try:
# use default formatting
msg = self.format(record)
if isinstance(msg, unicode):
msg = msg.encode("utf-8")
# produce message
if not self.key:
self.producer.send_messages(self.kafka_topic_name, msg)
else:
self.producer.send_messages(self.kafka_topic_name, self.key, msg)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
if self.producer is not None:
self.producer.stop()
logging.Handler.close(self)
