def get_config(self, cfg_type: str):
"""
根据cfg_type参数获取相应的zk配置
:param cfg_type:
:return:
"""
req_data = {
"username": self.base_config[cfg_type][self.zkUser],
"password": self.base_config[cfg_type][self.zkPassword],
"zkPath": self.base_config[cfg_type][self.zkNode]
}
# 请求ZooKeeper服务器, 获取相应的节点配置信息
try:
req = requests.post(ZkAddress, data=req_data)
resp = req.json()
except:
log.tag_error(
InitSer.InitZk,
"Get Zookeeper Info error, can't not get kafka config!")
raise ActionError("Zk Server Error, Init Kafka server error!!!")
if resp["status"] != ActionStatus.ActionSuccess:
log.tag_error(
InitSer.InitZk,
"Get Kafka ZkConfig Failed!, Error: %s" % resp["errMsg"])
raise ActionError(resp["errMsg"])
conf = resp["zkData"]
return conf
def offsets_for_time(self, partitions_time: list, timestamp: int = -1):
"""
寻找指定时间后的partition最早offset
:param partitions_time: list of (topic, partition) if timestamp > 0, (topic, partition, timestamp) if timestamp = -1
:param timestamp: 指定的开始查询时间, 如果是-1则表示每个partitions都有自己的时间配置
:return:
"""
if timestamp > 0:
_partitions = {
TopicPartition(_tuple[0], _tuple[1]): timestamp
for _tuple in partitions_time
}
else:
_partitions = {
TopicPartition(_tuple[0], _tuple[1]): _tuple[2]
for _tuple in partitions_time
}
try:
result = self.consumer.offsets_for_times(_partitions)
except UnsupportedVersionError or ValueError or KafkaTimeoutError as e:
if e.__class__ == UnsupportedVersionError:
log.tag_error(KafkaInfo.KafkaConsumer,
"API VERSION ERROR, DO NOT SUPPORT")
raise ActionError(KafkaErr.NotSupport)
if e.__class__ == ValueError:
log.tag_error(KafkaInfo.KafkaConsumer,
"Value Error: Target Timestamp is negative")
else:
log.tag_error(KafkaInfo.KafkaConsumer,
"Get offset by timestamp failed, Time out")
raise ActionError(KafkaErr.GetOffsetFailed)
return result
def inquire_record(self):
"""
log handle
:return:
"""
# html post of submit request
if self.request.method != "POST":
raise ActionError(UserActionErr.RequestErr)
data = self.request.form.to_dict()
# ajax post request
if "reqType" not in data.keys():
data = self.request.get_json()
data_keys = data.keys()
if "reqType" not in data_keys or "topic" not in data_keys:
raise ActionError(UserActionErr.ParamErr)
req_type = str(data["reqType"])
topic = data["topic"]
topics = self.get_role_topic()
msg_keys = self.get_msg_kye(topic)
self.context["topics"] = topics
self.context["currentTopic"] = topic
self.context["msgKeys"] = msg_keys
# if topic have no any message keys
if len(msg_keys) == 0:
self.context["currentMsgKey"] = ""
self.context[
"filterTime"] = "" if "filterTime" not in data_keys else data[
"filterTime"]
self.context["records"] = []
return render_template("record.html", **self.context)
# first inquire log of topic
if req_type == LogReq.TopicReq:
records = self.inquire_msg(topic, msg_keys[0])
self.context["currentMsgKey"] = msg_keys[0]
self.context["filterTime"] = ""
self.context["records"] = records
return render_template("record.html", **self.context)
# then choice log inquire of topic
if "currentMsgKey" not in data_keys or "filterTime" not in data_keys or "offset" not in data_keys:
raise ActionError(UserActionErr.ParamErr)
current_msg_key = data[
"currentMsgKey"] if data["currentMsgKey"] != "" else msg_keys[0]
filter_time = data["filterTime"]
offset = int(data["offset"])
if req_type != LogReq.TopicKeyReq:
raise ActionError(UserActionErr.ParamErr)
records = self.inquire_msg(topic, current_msg_key, offset, filter_time)
self.context["currentMsgKey"] = current_msg_key
self.context["filterTime"] = filter_time
self.context["records"] = records
# return jsonify(self.context)
return render_template("record.html", **self.context)
def __init__(self, request):
self.request = request
self.obj = self.request.get_json()
if "verify" not in self.obj.keys() or "timestamp" not in self.obj.keys(
) or "username" not in self.obj.keys():
raise ActionError(UserActionErr.ParamErr)
self.role = User.query.filter_by(name=self.obj["username"]).first()
if not self.role:
raise ActionError(UserActionErr.UserNotExist)
# verify action access
if self.obj["verify"] != md5_check(self.role.md5_id,
self.obj["timestamp"]):
raise ActionError(UserActionErr.RequestErr)
def __init__(self, request):
self.request = request
self.context = dict()
if not current_user.is_authenticated:
log.tag_info(WebLogTag.UserAccess, UserActionInfo.SessionExpire)
raise ActionError(UserActionInfo.SessionExpire)
self.user = get_user(current_user.id)
if self.user is None:
log.tag_error(
WebLogTag.UserAccess,
"Web session error, couldn't get user(%s) "
"when inquire log record" % str(current_user.id))
raise ActionError(WebInfo.SessionErr)
def _get_msg_by_key(self, consumer: Consumer, key: str, offset: int,
key_partition: int, need_num: int, timestamp: int):
"""
获取消息(倒序查询), 我们用key区分topic下的具体消息
:param consumer:
:param key:
:param offset:
:param key_partition:
:param need_num: need_num is the number of message that you need, if it's not enough, return direct.
:param timestamp: if timestamp not equal to 0, then all inquire records need recorded before timestamp
:return:
"""
key_byte = key.encode("utf-8")
not_enough = True
current_offset = offset
key_records = list()
while not_enough:
if current_offset <= self.ONCE_GET:
seek_position = 0
max_record = current_offset if current_offset != 0 else self.ONCE_GET
else:
seek_position = current_offset - self.ONCE_GET
max_record = self.ONCE_GET
# seek至指定position 开始poll record(顺序poll的, 因此需要处理下再返回)
consumer.seek((self.topic, key_partition), seek_position)
records = consumer.poll(timeout_ms=10000, max_records=max_record)
if records == {}:
log.tag_error(InquireErr.TimeOut,
"Inquire [%s] records failed, time out" % key)
raise ActionError(InquireErr.TimeOut)
records_value = records[consumer.get_topic_partition(
self.topic, key_partition)]
# 筛选出所需key的record
if timestamp == 0:
_records = [
_record for _record in records_value
if _record.key == key_byte
]
else:
_records = [
_record for _record in records_value if
_record.key == key_byte and _record.timestamp <= timestamp
]
# 该partition下所有records遍历完毕则退出
current_offset = current_offset - self.ONCE_GET
if current_offset <= 0:
not_enough = False
if len(_records) < need_num - len(key_records):
_records.extend(key_records)
key_records = copy.deepcopy(_records)
continue
else:
_records = _records[len(_records) - need_num +
len(key_records):]
_records.extend(key_records)
key_records = copy.deepcopy(_records)
not_enough = False
return key_records
def get_msg_with_offset(self,
key: str,
offset: int = 0,
need_num: int = 20,
timestamp: int = 0):
"""
获取消息(倒序查询), 我们用key区分topic下的具体消息
:param key:
:param offset:
:param need_num:
:param timestamp:
:return:
"""
key_byte = key.encode("utf-8")
with Consumer(self.group_id) as consumer:
# 获取key默认分配的分区
key_partition = self.get_default_partition_by_key(
key_byte, consumer)
partitions = [(self.topic, key_partition)]
consumer.assign(partitions)
# seek offset to the end
consumer.seek_many(partitions, is_begin=False)
last_offset = consumer.position(partitions[0])
if last_offset == 0:
return []
# 如果无记录则直接返回
if offset > last_offset:
raise ActionError(InquireErr.OffsetErr)
current_offset = offset
records = self._get_msg_by_key(consumer, key, current_offset,
key_partition, need_num, timestamp)
return records
def inquire_topic(self):
"""
inquire user topic
:return:
"""
if self.request.method != "GET":
raise ActionError(UserActionErr.RequestErr)
# inquire role topic
self.context["topics"] = self.get_role_topic()
return render_template("topic.html", **self.context)
def end_offsets(self, partitions: list):
"""
获取指定partition结束的offset
:param partitions: 指定topic分区[(topic: partition)]
:return:
"""
_partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions]
try:
result = self.consumer.end_offsets(_partitions)
except UnsupportedVersionError or KafkaTimeoutError as e:
if e.__class__ == UnsupportedVersionError:
log.tag_error(KafkaInfo.KafkaConsumer,
"API VERSION ERROR, DO NOT SUPPORT")
raise ActionError(KafkaErr.NotSupport)
else:
log.tag_error(KafkaInfo.KafkaConsumer,
"Get end offset failed, Time out")
raise ActionError(KafkaErr.GetOffsetFailed)
return result
def flush(self):
"""
调用此方法会使得所有缓存记录变成立即可发送状态.(一般用于send之后, 需要刷新)
:return:
"""
try:
self.producer.flush(timeout=TIME_OUT)
except KafkaTimeoutError:
log.tag_error(KafkaInfo.KafkaProducer,
"Flush buffered record failed, TimeOut")
raise ActionError(KafkaErr.FlushFailed)
def commit_err(topic: str, partition: int, offset: int):
"""
consumer commit offset failed callback function
:param topic:
:param partition:
:param offset:
:return:
"""
log.tag_error(
KafkaInfo.KafkaConsumer, "Kafka Consumer commit offset failed, "
"{TopicPartition(%s, %s): %s}" % (topic, partition, offset))
raise ActionError(KafkaErr.CommitOffsetFailed)
def commit():
"""
db session commit, will rollback when error occurred
:return:
"""
try:
db.session.commit()
except Exception as e:
db.session.rollback()
log.tag_error(WebLogTag.DbErr,
"Db Session Committed Err: %s" % e.__str__())
raise ActionError(UserActionErr.DataBaseErr)
def send_err(topic: str, value: bytes, key: str):
"""
producer send data failed callback function
:param topic:
:param value:
:param key:
:return:
:return:
"""
log.tag_error(
KafkaInfo.KafkaProducer, "Kafka send data failed, topic: %s, "
"key: %s msg: %s" % (topic, key, value.decode("utf-8")))
raise ActionError(KafkaErr.SendDataFailed)
def get_msg_kye(topic_name: str):
"""
inquire all topic's msg_key according to topic_name
:param topic_name:
:return:
"""
topic_info = TopicInfo.query.filter_by(topic_name=topic_name).first()
if not topic_info:
raise ActionError(WebInfo.TopicNotExist)
topic_key = TopicKey.query.filter_by(
topic_id=topic_info.topic_id).all()
msg_keys = [_key.msg_key for _key in topic_key]
return msg_keys
def mock_clear_topic(topic_name: str, client: ClientAdmin = None):
"""
删除模拟创建的topic
:return:
"""
if client is not None:
try:
client.delete_topic(topic_name)
print("clear mock topic")
except ActionError as e:
if e.message == KafkaErr.TopicNotExist:
return
raise ActionError(e.message)
else:
with ClientAdmin() as client:
try:
client.delete_topic(topic_name)
print("clear mock topic")
except ActionError as e:
if e.message == KafkaErr.TopicNotExist:
return
raise ActionError(e.message)
def mock_test_topic(topic_name: str, client: ClientAdmin = None):
"""
模拟创建topic
:return:
"""
if client:
try:
client.create_topic(topic_name)
print("mock topic")
except ActionError as e:
if e.message != KafkaErr.TopicExist:
print("mock topic failed")
raise ActionError("Mock Create Topic error")
else:
with ClientAdmin() as client:
try:
client.create_topic(topic_name)
print("mock topic")
except ActionError as e:
if e.message != KafkaErr.TopicExist:
print("mock topic failed")
raise ActionError("Mock Create Topic error")
def pause(self, partitions: list):
"""
挂起分区的请求(注意请求失败, 但可能有部分topic partition已经挂起)
:param partitions: list of TopicPartition need pause, for example: [(topic1, partition1), (topic2, partition2)]
:return:
"""
_partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions]
try:
self.consumer.pause(*_partitions)
except:
log.tag_error(
KafkaInfo.KafkaConsumer,
"Pause TopicPartition error, TopicPartition not exist")
raise ActionError(KafkaErr.TopicPartitionNotExist)
def str_to_datetime(time_str, raise_error=False):
"""
"2017-04-06 14:06:51" 字符串转日期时间对象
:param time_str:
:param raise_error:
:return:
"""
try:
dt = parser.parse(time_str)
except Exception as e:
if raise_error:
raise ActionError("时间字符串格式错误")
else:
raise e
return dt
def send_msg(self):
"""
send message
:return:
"""
if "topic" not in self.obj.keys() or "msgKey" not in self.obj.keys(
) or "msg" not in self.obj.keys():
raise ActionError(UserActionErr.ParamErr)
_topic = self.obj["topic"]
_msg_key = self.obj["msgKey"]
_msg = self.obj["msg"]
topic_info = TopicInfo.query.filter_by(topic_name=_topic).first()
if topic_info is None:
raise ActionError(UserActionErr.TopicNotExist)
# check topic is belongs to role or not
role_topics = RoleTopic.query.filter_by(role_id=self.role.id).all()
role_topic_ids = [_role_topic.topic_id for _role_topic in role_topics]
if topic_info.topic_id not in role_topic_ids:
raise ActionError(UserActionErr.TopicNotBelong)
# TODO 考虑是否启用批量send message
try:
send_msg(_topic, _msg_key, _msg)
except ActionError:
raise ActionError(UserActionErr.ActionFailed)
# check msgKey is record or not, if not record it
topic_key = TopicKey.query.filter_by(msg_key=_msg_key).first()
if topic_key is None:
key = TopicKey(topic_id=topic_info.topic_id, msg_key=_msg_key)
db.session.add(key)
commit()
res = {
"errCode": ResponseStatus.Success,
"errMsg": ResponseMsg[ResponseStatus.Success],
"obj": {}
}
return jsonify(res)
def delete_topic(self, topic_name: str):
"""
删除集群中的topic
:param topic_name:
:return:
"""
topic_list = [topic_name]
try:
self.admin_client.delete_topics(topic_list,
timeout_ms=TIME_OUT_ADMIN)
except UnknownTopicOrPartitionError as e:
log.tag_error(
KafkaInfo.KafkaAdmin,
"Topic [%s] not exist! Don't need delete" % topic_name)
raise ActionError(KafkaErr.TopicNotExist)
def create_topic(self, topic_name: str):
"""
在集群中创建新的topic(topic配置采用默认模式)
:param topic_name:
:return:
"""
topic_list = [self.new_topic(topic_name)]
try:
response = self.admin_client.create_topics(
topic_list, timeout_ms=TIME_OUT_ADMIN)
except TopicAlreadyExistsError:
log.tag_error(
KafkaInfo.KafkaAdmin,
"Topic [%s] already exist! Create Failed !" % topic_name)
raise ActionError(KafkaErr.TopicExist)
return response
def subscribe(self, topic: list, pattern: str = None):
"""
订阅一组topics
:param topic: topic 列表
:param pattern:
:return:
"""
try:
self.consumer.subscribe(topic, pattern)
except IllegalStateError or AssertionError or TypeError as e:
if e.__class__ == IllegalStateError:
log.tag_error(KafkaInfo.KafkaConsumer,
"Subscribe topic error, %s" % e.__str__)
log.tag_error(KafkaInfo.KafkaConsumer,
"Subscribe topic error, Parameter Error")
raise ActionError(KafkaErr.ParameterError)
def assign(self, partitions: list):
"""
手动为当前consumer分配topic分区列表
:param partitions: 手动分配分区[(topic, partition)]
:return:
"""
_partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions]
try:
result = self.consumer.assign(_partitions)
except IllegalStateError:
log.tag_error(
KafkaInfo.KafkaConsumer,
"Manually consumer TopicPartitions error, "
"Topic Consumer is being in used")
raise ActionError(KafkaErr.ConsumerInUsed)
return result
def get_kafka_cfg():
"""
获取kafka访问配置
:return:
"""
try:
# 使用zk获取配置
cfg = ZkConfig("test", "ts1245", "/kafka").get_config(KaCfgType.KafkaCfg)
except ActionError:
# zk服务不可用时, 采用本地config配置
cfg = KafkaConfig
# cfg 中的apiVersion需要单独处理
try:
cfg["apiVersion"] = tuple(cfg["apiVersion"])
except Exception:
raise ActionError("Kafka config error!!!")
return cfg
def register(self):
"""
用户注册
:return:
"""
if self.request.method == "GET":
return render_template("register.html", **self.context)
if self.request.method != "POST":
raise ActionError(UserActionErr.RequestErr)
par_dict = self.request.form.to_dict()
user_name = par_dict["user_name"]
password = par_dict["password"]
email = par_dict["email"]
sex = par_dict["sex"]
# check name has been registered or not
record = get_username(user_name)
if record is not None:
self.context["errMsg"] = UserActionErr.UserExist
return render_template("register.html", **self.context)
# check password
if not pass_word_check(password):
self.context["errMsg"] = UserActionErr.PasswordFormatErr
return render_template("register.html", **self.context)
# register user
_password = generate_md5(password)
user = User(name=user_name,
password_hash=_password,
email=email,
sex=sex)
db.session.add(user)
commit()
log.tag_info(
WebLogTag.UserAccess,
"Register Success, User: %s, email: %s, sex: %d" %
(user.name, user.email, user.sex))
self.user_login(user)
self.context["errMsg"] = UserActionInfo.RegisterSuccess + user_name
return render_template("register.html", **self.context)
def login(self):
"""
用户登录
:return:
"""
self.context["Check"] = ""
if self.request.method == "GET":
return render_template("login.html", **self.context)
if self.request.method != "POST":
raise ActionError(UserActionErr.RequestErr)
par_dict = self.request.form.to_dict()
user_name = par_dict["user_name"]
password = par_dict['password']
user = get_username(user_name)
if not user:
self.context["Check"] = UserActionErr.LoginErr
return render_template("login.html", **self.context)
if user.verity_password(password):
self.user_login(user)
self.context["Check"] = UserActionInfo.LoginSuccess + user_name
return render_template("login.html", **self.context)
def send_message(self, topic_name: str, value: bytes, key: str):
"""
Producer产生数据
:param topic_name: topic where the message will be published
:param value: message value
:param key: key to associate with the message
:return:
"""
try:
result = self.producer.send(topic_name,
value=value,
key=key.encode("utf-8")).add_errback(
self.send_err,
topic=topic_name,
value=value,
key=key)
except KafkaTimeoutError:
log.tag_warn(
KafkaInfo.KafkaProducer,
"Kafka send data timeout, topic: %s, key: %s, msg: %s" %
(topic_name, key, value.decode("utf-8")))
raise ActionError(KafkaErr.SendDataFailed)
return result