def Consumer():
data = []
start_time=timer()
name=multiprocessing.current_process().name
# print(name,'Starting')
while True:
print(name,'Starting')
consumer = KafkaConsumer('topic-weather-stations',group_id='consumer-weather-data',bootstrap_servers=['vm1:9092'],consumer_timeout_ms=15000,heartbeat_interval_ms=1000)
consumer.zookeeper_connect='vm1:2181'
try:
for message in consumer:
data.append(message.value)
if len(data) >15000:
insert_weather_stations(data,name)
data=[]
else:
continue
finally:
print(name,'Exiting now')
if len(data) >0:
insert_weather_stations(data,name)
data=[]
sys.stdout.flush()
consumer.close()
def Consumer():
data = []
start_time=timer()
name=multiprocessing.current_process().name
while True:
print (name,'Starting')
consumer = KafkaConsumer('topic-weather-data',group_id='consumer-weather-data',bootstrap_servers=['vm1:9092'],consumer_timeout_ms=14000,heartbeat_interval_ms=1000)
consumer.zookeeper_connect='vm1:2181'
try:
for message in consumer:
data.append(message.value)
if len(data) >5000:
insert_raw_data(data,name)
# collect_data(data)
data=[]
else:
continue
finally:
print(name,'Exiting now',len(data))
if len(data) >0:
try:
insert_raw_data(data,name)
# collect_data(data)
data=[]
except Exception,e :
print('Error due to ',e)
sys.stdout.flush()
print (name,'Closing out',timer() - start_time)
consumer.close()
def Consumer():
global data
start_time=timer()
consumer = KafkaConsumer('temp',group_id='consumer-temp',bootstrap_servers=['vm1:9092'],consumer_timeout_ms=20000,heartbeat_interval_ms=1000)
# consumer.subscribe('temp')
consumer.zookeeper_connect='localhost:2181'
try:
for message in consumer:
data.append(message.value)
#time.sleep(3)
if len(data) >5000:
kafka_insert_data.insert_vals(data)
data=[]
else:
continue
#print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,message.offset, message.key, message.value))
finally:
print('Exiting now')
if len(data) >0:
kafka_insert_data.insert_vals(data)
data=[]
#consumer.commit_async()
consumer.close()
