def _get_msg_by_key(self, consumer: Consumer, key: str, offset: int,
key_partition: int, need_num: int, timestamp: int):
"""
获取消息(倒序查询), 我们用key区分topic下的具体消息
:param consumer:
:param key:
:param offset:
:param key_partition:
:param need_num: need_num is the number of message that you need, if it's not enough, return direct.
:param timestamp: if timestamp not equal to 0, then all inquire records need recorded before timestamp
:return:
"""
key_byte = key.encode("utf-8")
not_enough = True
current_offset = offset
key_records = list()
while not_enough:
if current_offset <= self.ONCE_GET:
seek_position = 0
max_record = current_offset if current_offset != 0 else self.ONCE_GET
else:
seek_position = current_offset - self.ONCE_GET
max_record = self.ONCE_GET
# seek至指定position 开始poll record(顺序poll的, 因此需要处理下再返回)
consumer.seek((self.topic, key_partition), seek_position)
records = consumer.poll(timeout_ms=10000, max_records=max_record)
if records == {}:
log.tag_error(InquireErr.TimeOut,
"Inquire [%s] records failed, time out" % key)
raise ActionError(InquireErr.TimeOut)
records_value = records[consumer.get_topic_partition(
self.topic, key_partition)]
# 筛选出所需key的record
if timestamp == 0:
_records = [
_record for _record in records_value
if _record.key == key_byte
]
else:
_records = [
_record for _record in records_value if
_record.key == key_byte and _record.timestamp <= timestamp
]
# 该partition下所有records遍历完毕则退出
current_offset = current_offset - self.ONCE_GET
if current_offset <= 0:
not_enough = False
if len(_records) < need_num - len(key_records):
_records.extend(key_records)
key_records = copy.deepcopy(_records)
continue
else:
_records = _records[len(_records) - need_num +
len(key_records):]
_records.extend(key_records)
key_records = copy.deepcopy(_records)
not_enough = False
return key_records
def get_config(self, cfg_type: str):
"""
根据cfg_type参数获取相应的zk配置
:param cfg_type:
:return:
"""
req_data = {
"username": self.base_config[cfg_type][self.zkUser],
"password": self.base_config[cfg_type][self.zkPassword],
"zkPath": self.base_config[cfg_type][self.zkNode]
}
# 请求ZooKeeper服务器, 获取相应的节点配置信息
try:
req = requests.post(ZkAddress, data=req_data)
resp = req.json()
except:
log.tag_error(
InitSer.InitZk,
"Get Zookeeper Info error, can't not get kafka config!")
raise ActionError("Zk Server Error, Init Kafka server error!!!")
if resp["status"] != ActionStatus.ActionSuccess:
log.tag_error(
InitSer.InitZk,
"Get Kafka ZkConfig Failed!, Error: %s" % resp["errMsg"])
raise ActionError(resp["errMsg"])
conf = resp["zkData"]
return conf
def flush(self):
"""
调用此方法会使得所有缓存记录变成立即可发送状态.(一般用于send之后, 需要刷新)
:return:
"""
try:
self.producer.flush(timeout=TIME_OUT)
except KafkaTimeoutError:
log.tag_error(KafkaInfo.KafkaProducer,
"Flush buffered record failed, TimeOut")
raise ActionError(KafkaErr.FlushFailed)
def commit():
"""
db session commit, will rollback when error occurred
:return:
"""
try:
db.session.commit()
except Exception as e:
db.session.rollback()
log.tag_error(WebLogTag.DbErr,
"Db Session Committed Err: %s" % e.__str__())
raise ActionError(UserActionErr.DataBaseErr)
def commit_err(topic: str, partition: int, offset: int):
"""
consumer commit offset failed callback function
:param topic:
:param partition:
:param offset:
:return:
"""
log.tag_error(
KafkaInfo.KafkaConsumer, "Kafka Consumer commit offset failed, "
"{TopicPartition(%s, %s): %s}" % (topic, partition, offset))
raise ActionError(KafkaErr.CommitOffsetFailed)
def send_err(topic: str, value: bytes, key: str):
"""
producer send data failed callback function
:param topic:
:param value:
:param key:
:return:
:return:
"""
log.tag_error(
KafkaInfo.KafkaProducer, "Kafka send data failed, topic: %s, "
"key: %s msg: %s" % (topic, key, value.decode("utf-8")))
raise ActionError(KafkaErr.SendDataFailed)
def __init__(self, request):
self.request = request
self.context = dict()
if not current_user.is_authenticated:
log.tag_info(WebLogTag.UserAccess, UserActionInfo.SessionExpire)
raise ActionError(UserActionInfo.SessionExpire)
self.user = get_user(current_user.id)
if self.user is None:
log.tag_error(
WebLogTag.UserAccess,
"Web session error, couldn't get user(%s) "
"when inquire log record" % str(current_user.id))
raise ActionError(WebInfo.SessionErr)
def pause(self, partitions: list):
"""
挂起分区的请求(注意请求失败, 但可能有部分topic partition已经挂起)
:param partitions: list of TopicPartition need pause, for example: [(topic1, partition1), (topic2, partition2)]
:return:
"""
_partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions]
try:
self.consumer.pause(*_partitions)
except:
log.tag_error(
KafkaInfo.KafkaConsumer,
"Pause TopicPartition error, TopicPartition not exist")
raise ActionError(KafkaErr.TopicPartitionNotExist)
def delete_topic(self, topic_name: str):
"""
删除集群中的topic
:param topic_name:
:return:
"""
topic_list = [topic_name]
try:
self.admin_client.delete_topics(topic_list,
timeout_ms=TIME_OUT_ADMIN)
except UnknownTopicOrPartitionError as e:
log.tag_error(
KafkaInfo.KafkaAdmin,
"Topic [%s] not exist! Don't need delete" % topic_name)
raise ActionError(KafkaErr.TopicNotExist)
def assign(self, partitions: list):
"""
手动为当前consumer分配topic分区列表
:param partitions: 手动分配分区[(topic, partition)]
:return:
"""
_partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions]
try:
result = self.consumer.assign(_partitions)
except IllegalStateError:
log.tag_error(
KafkaInfo.KafkaConsumer,
"Manually consumer TopicPartitions error, "
"Topic Consumer is being in used")
raise ActionError(KafkaErr.ConsumerInUsed)
return result
def subscribe(self, topic: list, pattern: str = None):
"""
订阅一组topics
:param topic: topic 列表
:param pattern:
:return:
"""
try:
self.consumer.subscribe(topic, pattern)
except IllegalStateError or AssertionError or TypeError as e:
if e.__class__ == IllegalStateError:
log.tag_error(KafkaInfo.KafkaConsumer,
"Subscribe topic error, %s" % e.__str__)
log.tag_error(KafkaInfo.KafkaConsumer,
"Subscribe topic error, Parameter Error")
raise ActionError(KafkaErr.ParameterError)
def create_topic(self, topic_name: str):
"""
在集群中创建新的topic(topic配置采用默认模式)
:param topic_name:
:return:
"""
topic_list = [self.new_topic(topic_name)]
try:
response = self.admin_client.create_topics(
topic_list, timeout_ms=TIME_OUT_ADMIN)
except TopicAlreadyExistsError:
log.tag_error(
KafkaInfo.KafkaAdmin,
"Topic [%s] already exist! Create Failed !" % topic_name)
raise ActionError(KafkaErr.TopicExist)
return response
def end_offsets(self, partitions: list):
"""
获取指定partition结束的offset
:param partitions: 指定topic分区[(topic: partition)]
:return:
"""
_partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions]
try:
result = self.consumer.end_offsets(_partitions)
except UnsupportedVersionError or KafkaTimeoutError as e:
if e.__class__ == UnsupportedVersionError:
log.tag_error(KafkaInfo.KafkaConsumer,
"API VERSION ERROR, DO NOT SUPPORT")
raise ActionError(KafkaErr.NotSupport)
else:
log.tag_error(KafkaInfo.KafkaConsumer,
"Get end offset failed, Time out")
raise ActionError(KafkaErr.GetOffsetFailed)
return result
def offsets_for_time(self, partitions_time: list, timestamp: int = -1):
"""
寻找指定时间后的partition最早offset
:param partitions_time: list of (topic, partition) if timestamp > 0, (topic, partition, timestamp) if timestamp = -1
:param timestamp: 指定的开始查询时间, 如果是-1则表示每个partitions都有自己的时间配置
:return:
"""
if timestamp > 0:
_partitions = {
TopicPartition(_tuple[0], _tuple[1]): timestamp
for _tuple in partitions_time
}
else:
_partitions = {
TopicPartition(_tuple[0], _tuple[1]): _tuple[2]
for _tuple in partitions_time
}
try:
result = self.consumer.offsets_for_times(_partitions)
except UnsupportedVersionError or ValueError or KafkaTimeoutError as e:
if e.__class__ == UnsupportedVersionError:
log.tag_error(KafkaInfo.KafkaConsumer,
"API VERSION ERROR, DO NOT SUPPORT")
raise ActionError(KafkaErr.NotSupport)
if e.__class__ == ValueError:
log.tag_error(KafkaInfo.KafkaConsumer,
"Value Error: Target Timestamp is negative")
else:
log.tag_error(KafkaInfo.KafkaConsumer,
"Get offset by timestamp failed, Time out")
raise ActionError(KafkaErr.GetOffsetFailed)
return result
