def start(self):
kafka_brokers = '{0}:{1}'.format(self._server, self._port)
consumer = KC(bootstrap_servers=[kafka_brokers], group_id=self._topic)
partition = [TopicPartition(self._topic, int(self._id))]
consumer.assign(partitions=partition)
consumer.poll()
return consumer
def getStreamDataFromKafka(topic, partition, offset):
consumer = KC(bootstrap_servers=['MN:9092'])
print('topic:', topic, 'partition:', partition, 'offset', offset)
topicPartition = TP(topic, partition)
consumer.assign([topicPartition])
consumer.seek(topicPartition, offset)
testDataArray = []
for message in consumer:
testDataArray.append(message.value.decode('utf-8'))
filePath = '/data/' + str(topic) + '_' + str(offset) + '.csv'
if len(testDataArray) == 10240:
file = open(filePath, 'w')
for data in testDataArray:
file.write(data + '\n')
file.close()
break
return filePath
def build_kafka_consumer(
topic: str,
value_deserializer=lambda v: json.loads(v.decode("utf-8")),
auto_offset_reset="latest",
test=TEST):
if test:
consumer = TestKafkaConsumer(topic=topic,
auto_offset_reset=auto_offset_reset,
max=10)
elif KAFKA_BASE64_KEYS:
consumer = helper.get_kafka_consumer(
topic=topic,
auto_offset_reset=auto_offset_reset,
value_deserializer=value_deserializer)
else:
consumer = KC(
topic,
bootstrap_servers=KAFKA_HOSTS,
auto_offset_reset=auto_offset_reset,
value_deserializer=value_deserializer,
)
return consumer