def Consumer(): data = [] start_time=timer() name=multiprocessing.current_process().name # print(name,'Starting') while True: print(name,'Starting') consumer = KafkaConsumer('topic-weather-stations',group_id='consumer-weather-data',bootstrap_servers=['vm1:9092'],consumer_timeout_ms=15000,heartbeat_interval_ms=1000) consumer.zookeeper_connect='vm1:2181' try: for message in consumer: data.append(message.value) if len(data) >15000: insert_weather_stations(data,name) data=[] else: continue finally: print(name,'Exiting now') if len(data) >0: insert_weather_stations(data,name) data=[] sys.stdout.flush() consumer.close()
def Consumer(): data = [] start_time=timer() name=multiprocessing.current_process().name while True: print (name,'Starting') consumer = KafkaConsumer('topic-weather-data',group_id='consumer-weather-data',bootstrap_servers=['vm1:9092'],consumer_timeout_ms=14000,heartbeat_interval_ms=1000) consumer.zookeeper_connect='vm1:2181' try: for message in consumer: data.append(message.value) if len(data) >5000: insert_raw_data(data,name) # collect_data(data) data=[] else: continue finally: print(name,'Exiting now',len(data)) if len(data) >0: try: insert_raw_data(data,name) # collect_data(data) data=[] except Exception,e : print('Error due to ',e) sys.stdout.flush() print (name,'Closing out',timer() - start_time) consumer.close()
def Consumer(): global data start_time=timer() consumer = KafkaConsumer('temp',group_id='consumer-temp',bootstrap_servers=['vm1:9092'],consumer_timeout_ms=20000,heartbeat_interval_ms=1000) # consumer.subscribe('temp') consumer.zookeeper_connect='localhost:2181' try: for message in consumer: data.append(message.value) #time.sleep(3) if len(data) >5000: kafka_insert_data.insert_vals(data) data=[] else: continue #print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,message.offset, message.key, message.value)) finally: print('Exiting now') if len(data) >0: kafka_insert_data.insert_vals(data) data=[] #consumer.commit_async() consumer.close()