class KafkaLfProducer(object): def __init__(self, addr, conf_file, start_house_id, end_house_id, house_status): self.parser = SafeConfigParser() self.parser.read(conf_file) install_dir = self.parser.get('smw_tool', 'INSTALL_DIR') zipdb_file = self.parser.get('smw_tool', 'ZIP_DB_FILE') self.client = KafkaClient(addr) self.producer = KeyedProducer(self.client, async=True, batch_send_every_n=500,batch_send=True) self.meterReader = MeterLfReader(start_house_id, end_house_id, house_status, install_dir + "/data/low_freq/", install_dir + "/" + zipdb_file) def produce_msgs(self, source_symbol): msg_cnt = 0 while not self.meterReader.houseSentDone(): (isLf, msg) = self.meterReader.getRecord() if msg_cnt % 500000 == 0: print "Sent " + str(msg_cnt) + " messages to Kafka" if isLf: self.producer.send_messages('smw_batch_lf2', source_symbol, msg) else: self.producer.send_messages('smw_batch_hf2', source_symbol, msg) msg_cnt += 1 print "Sent Total " + str(msg_cnt) + " messages to Kafka" self.meterReader.writeHouseStatus()
def __init__(self, addr, conf_file, start_house_id, end_house_id, house_status): self.parser = SafeConfigParser() self.parser.read(conf_file) install_dir = self.parser.get('smw_tool', 'INSTALL_DIR') zipdb_file = self.parser.get('smw_tool', 'ZIP_DB_FILE') self.client = KafkaClient(addr) self.producer = KeyedProducer(self.client, async=True, batch_send_every_n=500,batch_send=True) self.meterReader = MeterLfReader(start_house_id, end_house_id, house_status, install_dir + "/data/low_freq/", install_dir + "/" + zipdb_file)