def main():
# 1.获取Kafka对象
kafka = Kafka(group_id="DP1")
# 2.读取Kafka数据
# consumer = kafka.read_kafka(CONSUMER_TOPIC)
for i in range(100):
print(i)
kafka.write_kafka(CONSUMER_TOPIC, f"message_{i}")
print("end")
def write_kafka():
topic = request.args.get("topic")
message = request.args.get("message")
if not topic or not message:
return to_dict_msg(
status=200,
msg="write kafka failed, topic and message can not empty")
try:
kafka = Kafka()
kafka.write_kafka(topic, json.loads(message))
return to_dict_msg(status=200, msg="write kafka success")
except Exception as e:
print(e)
return to_dict_msg(status=500)
def main():
# 1.获取Kafka对象
kafka = Kafka()
# 2.读取Kafka数据
consumer = kafka.read_kafka(CONSUMER_TOPIC)
# 3.获取HBase对象
hbase = HBase()
# 4.创建HBase表
hbase.get_create_table(TARGET_TABLE_NAME, COLUMN_FAMILY)
for message in consumer:
data = json.loads(message.value)
if data['table'] == SOURCE_TABLE_NAME:
obj = data['data'][0]
row_key = obj[ROW_KEY]
# 5.写入HBase
hbase.put_row(TARGET_TABLE_NAME, COLUMN_FAMILY, row_key, dict(obj))
# 6.写入Kafka
message = json.dumps({TARGET_TABLE_NAME: obj})
kafka.write_kafka(PRODUCER_TOPIC, message)