class KafkaLoggingHandler(logging.Handler): def __init__(self, hosts_list, topic, key=None): logging.Handler.__init__(self) self.kafka_client = KafkaClient(hosts_list) self.key = key self.kafka_topic_name = topic if not key: self.producer = SimpleProducer(self.kafka_client) else: self.producer = KeyedProducer(self.kafka_client) def emit(self, record): # drop kafka logging to avoid infinite recursion if record.name == 'kafka': return try: # use default formatting msg = self.format(record) # produce message if not self.key: self.producer.send_messages(self.kafka_topic_name, msg) else: self.producer.send(self.kafka_topic_name, self.key, msg) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self.producer.stop() logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler): def __init__(self, hosts="", topic="", partition=0): logging.Handler.__init__(self) self.kafkaClient = KafkaClient(hosts) self.topic = topic self.partition = partition self.producer = KeyedProducer( self.kafkaClient, async=False, req_acks=KeyedProducer.ACK_AFTER_LOCAL_WRITE, ack_timeout=200 ) def emit(self, record): # drop kafka logging to avoid infinite recursion if record.name == "kafka": return try: # use default formatting msg = self.format(record) # produce message self.producer.send_messages(self.topic + record.name, self.partition, msg) except: import traceback ei = sys.exc_info() traceback.print_exception(ei[0], ei[1], ei[2], None, sys.stderr) del ei def close(self): self.producer.stop() logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler): def __init__(self, hosts_list, topic, key=None): logging.Handler.__init__(self) self.kafka_client = KafkaClient(hosts_list) self.key = key self.kafka_topic_name = topic if not key: self.producer = SimpleProducer(self.kafka_client) else: self.producer = KeyedProducer(self.kafka_client) def emit(self, record): # drop kafka logging to avoid infinite recursion if record.name == 'kafka': return try: # use default formatting msg = self.format(record) if isinstance(msg, unicode): msg = msg.encode("utf-8") # produce message if not self.key: self.producer.send_messages(self.kafka_topic_name, msg) else: self.producer.send(self.kafka_topic_name, self.key, msg) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self.producer.stop() logging.Handler.close(self)
def offsetCommit(): global users checkUserPartitionMapping() kafkaClient = KafkaClient(kafkaHost, timeout=None) producer = KeyedProducer(kafkaClient, async=False, req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE, ack_timeout=200) for partition in partitions: encodedMessage = simplejson.dumps({'turtleName':turtleName, 'user':'', 'operation':'offsetCommit'}) print producer.send(kafkaTopic, partition, encodedMessage) producer.stop(1) kafkaClient.close()
class KafkaLoggingHandler(logging.Handler): """ Use kafka to send msg to elk platform """ def __init__(self, hosts_list, topic, timeout_secs=DEFAULT_SOCKET_TIMEOUT_SECONDS, **kwargs): logging.Handler.__init__(self) self.kafka_client = KafkaClient(hosts_list, timeout=timeout_secs) self.key = kwargs.get("key", None) self.kafka_topic_name = topic if not self.key: self.producer = SimpleProducer(self.kafka_client, **kwargs) else: self.producer = KeyedProducer(self.kafka_client, **kwargs) def emit(self, record): """ emit record :param record: :return: """ # drop kafka logging to avoid infinite recursion if record.name == 'kafka': return try: # use default formatting msg = self.format(record) msg = msg.encode("utf-8") # produce message if not self.key: self.producer.send_messages(self.kafka_topic_name, msg) else: self.producer.send_messages( self.kafka_topic_name, self.key, msg) except (KeyboardInterrupt, SystemExit): raise except BaseException: self.handleError(record) def close(self): """ close the client :return: """ if self.producer is not None: self.producer.stop() logging.Handler.close(self)
class Producer(object): def __init__(self, addr): self.client = KafkaClient(addr) self.producer = KeyedProducer(self.client) def open_save(self, fileName): log_file = open(fileName, "w") log_file.close() return log_file def create_topic(self, topic): script = "/usr/local/kafka/bin/kafka-topics.sh" os.system("{} --create --zookeeper localhost:2181 --topic {} --partitions {} --replication-factor 2".format(script, topic, "4")) return "topic {} created".format(topic) def produce_msgs(self, source_symbol, topic): server_topics = self.client.topic_partitions if topic not in server_topics: self.create_topic(topic) price_field = random.randint(800,1400) cities = ["Barcelona", "Philadelphia", "Honolulu", "Atlanta", "Miami", "Chicago", "SF", "LA", "NYC", "Houston", "Paris", "London", "Tokyo"] msg_cnt = 0 log_file = open("input1/{}.csv".format(topic), "a") while True: time_field = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f') location_field = random.choice(cities) price_field += random.randint(-10, 10)/10.0 str_fmt = "{},{},{},{}" message_info = str_fmt.format(source_symbol, time_field, location_field, price_field) print message_info log_file.write("{}\n".format(message_info)) self.producer.send_messages(topic, source_symbol, message_info) msg_cnt += 1 if msg_cnt > 200000: log_file.close() self.producer.stop() break
class KafkaLoggingHandler(logging.Handler): def __init__(self, hosts_list, topic, **kwargs): logging.Handler.__init__(self) self.kafka_client = SimpleClient(hosts_list) self.key = kwargs.get("key", None) self.kafka_topic_name = topic if not self.key: self.producer = SimpleProducer(self.kafka_client, **kwargs) else: self.producer = KeyedProducer(self.kafka_client, **kwargs) def emit(self, record): # drop kafka logging to avoid infinite recursion if record.name == 'kafka': return try: # use default formatting msg = self.format(record) if isinstance(msg, unicode): msg = msg.encode("utf-8") # produce message if not self.key: self.producer.send_messages(self.kafka_topic_name, msg) else: self.producer.send_messages(self.kafka_topic_name, self.key, msg) except (KeyboardInterrupt, SystemExit): raise except Exception: self.handleError(record) def close(self): if self.producer is not None: self.producer.stop() logging.Handler.close(self)
class KafkaLoggingHandler(logging.Handler): def __init__(self, hosts_list, topic, timeout_secs=DEFAULT_SOCKET_TIMEOUT_SECONDS, **kwargs): logging.Handler.__init__(self) self.kafka_client = KafkaClient(hosts_list, timeout=timeout_secs) self.key = kwargs.get("key", None) self.kafka_topic_name = topic if not self.key: self.producer = SimpleProducer(self.kafka_client, **kwargs) else: self.producer = KeyedProducer(self.kafka_client, **kwargs) self.addFilter(KafkaLoggingFilter()) def emit(self, record): try: # use default formatting msg = self.format(record) if isinstance(msg, unicode): msg = msg.encode("utf-8") # produce message if not self.key: self.producer.send_messages(self.kafka_topic_name, msg) else: self.producer.send_messages(self.kafka_topic_name, self.key, msg) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): if self.producer is not None: self.producer.stop() logging.Handler.close(self)