def _get_msg_by_key(self, consumer: Consumer, key: str, offset: int, key_partition: int, need_num: int, timestamp: int): """ 获取消息(倒序查询), 我们用key区分topic下的具体消息 :param consumer: :param key: :param offset: :param key_partition: :param need_num: need_num is the number of message that you need, if it's not enough, return direct. :param timestamp: if timestamp not equal to 0, then all inquire records need recorded before timestamp :return: """ key_byte = key.encode("utf-8") not_enough = True current_offset = offset key_records = list() while not_enough: if current_offset <= self.ONCE_GET: seek_position = 0 max_record = current_offset if current_offset != 0 else self.ONCE_GET else: seek_position = current_offset - self.ONCE_GET max_record = self.ONCE_GET # seek至指定position 开始poll record(顺序poll的, 因此需要处理下再返回) consumer.seek((self.topic, key_partition), seek_position) records = consumer.poll(timeout_ms=10000, max_records=max_record) if records == {}: log.tag_error(InquireErr.TimeOut, "Inquire [%s] records failed, time out" % key) raise ActionError(InquireErr.TimeOut) records_value = records[consumer.get_topic_partition( self.topic, key_partition)] # 筛选出所需key的record if timestamp == 0: _records = [ _record for _record in records_value if _record.key == key_byte ] else: _records = [ _record for _record in records_value if _record.key == key_byte and _record.timestamp <= timestamp ] # 该partition下所有records遍历完毕则退出 current_offset = current_offset - self.ONCE_GET if current_offset <= 0: not_enough = False if len(_records) < need_num - len(key_records): _records.extend(key_records) key_records = copy.deepcopy(_records) continue else: _records = _records[len(_records) - need_num + len(key_records):] _records.extend(key_records) key_records = copy.deepcopy(_records) not_enough = False return key_records
def get_config(self, cfg_type: str): """ 根据cfg_type参数获取相应的zk配置 :param cfg_type: :return: """ req_data = { "username": self.base_config[cfg_type][self.zkUser], "password": self.base_config[cfg_type][self.zkPassword], "zkPath": self.base_config[cfg_type][self.zkNode] } # 请求ZooKeeper服务器, 获取相应的节点配置信息 try: req = requests.post(ZkAddress, data=req_data) resp = req.json() except: log.tag_error( InitSer.InitZk, "Get Zookeeper Info error, can't not get kafka config!") raise ActionError("Zk Server Error, Init Kafka server error!!!") if resp["status"] != ActionStatus.ActionSuccess: log.tag_error( InitSer.InitZk, "Get Kafka ZkConfig Failed!, Error: %s" % resp["errMsg"]) raise ActionError(resp["errMsg"]) conf = resp["zkData"] return conf
def flush(self): """ 调用此方法会使得所有缓存记录变成立即可发送状态.(一般用于send之后, 需要刷新) :return: """ try: self.producer.flush(timeout=TIME_OUT) except KafkaTimeoutError: log.tag_error(KafkaInfo.KafkaProducer, "Flush buffered record failed, TimeOut") raise ActionError(KafkaErr.FlushFailed)
def commit(): """ db session commit, will rollback when error occurred :return: """ try: db.session.commit() except Exception as e: db.session.rollback() log.tag_error(WebLogTag.DbErr, "Db Session Committed Err: %s" % e.__str__()) raise ActionError(UserActionErr.DataBaseErr)
def commit_err(topic: str, partition: int, offset: int): """ consumer commit offset failed callback function :param topic: :param partition: :param offset: :return: """ log.tag_error( KafkaInfo.KafkaConsumer, "Kafka Consumer commit offset failed, " "{TopicPartition(%s, %s): %s}" % (topic, partition, offset)) raise ActionError(KafkaErr.CommitOffsetFailed)
def send_err(topic: str, value: bytes, key: str): """ producer send data failed callback function :param topic: :param value: :param key: :return: :return: """ log.tag_error( KafkaInfo.KafkaProducer, "Kafka send data failed, topic: %s, " "key: %s msg: %s" % (topic, key, value.decode("utf-8"))) raise ActionError(KafkaErr.SendDataFailed)
def __init__(self, request): self.request = request self.context = dict() if not current_user.is_authenticated: log.tag_info(WebLogTag.UserAccess, UserActionInfo.SessionExpire) raise ActionError(UserActionInfo.SessionExpire) self.user = get_user(current_user.id) if self.user is None: log.tag_error( WebLogTag.UserAccess, "Web session error, couldn't get user(%s) " "when inquire log record" % str(current_user.id)) raise ActionError(WebInfo.SessionErr)
def pause(self, partitions: list): """ 挂起分区的请求(注意请求失败, 但可能有部分topic partition已经挂起) :param partitions: list of TopicPartition need pause, for example: [(topic1, partition1), (topic2, partition2)] :return: """ _partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions] try: self.consumer.pause(*_partitions) except: log.tag_error( KafkaInfo.KafkaConsumer, "Pause TopicPartition error, TopicPartition not exist") raise ActionError(KafkaErr.TopicPartitionNotExist)
def delete_topic(self, topic_name: str): """ 删除集群中的topic :param topic_name: :return: """ topic_list = [topic_name] try: self.admin_client.delete_topics(topic_list, timeout_ms=TIME_OUT_ADMIN) except UnknownTopicOrPartitionError as e: log.tag_error( KafkaInfo.KafkaAdmin, "Topic [%s] not exist! Don't need delete" % topic_name) raise ActionError(KafkaErr.TopicNotExist)
def assign(self, partitions: list): """ 手动为当前consumer分配topic分区列表 :param partitions: 手动分配分区[(topic, partition)] :return: """ _partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions] try: result = self.consumer.assign(_partitions) except IllegalStateError: log.tag_error( KafkaInfo.KafkaConsumer, "Manually consumer TopicPartitions error, " "Topic Consumer is being in used") raise ActionError(KafkaErr.ConsumerInUsed) return result
def subscribe(self, topic: list, pattern: str = None): """ 订阅一组topics :param topic: topic 列表 :param pattern: :return: """ try: self.consumer.subscribe(topic, pattern) except IllegalStateError or AssertionError or TypeError as e: if e.__class__ == IllegalStateError: log.tag_error(KafkaInfo.KafkaConsumer, "Subscribe topic error, %s" % e.__str__) log.tag_error(KafkaInfo.KafkaConsumer, "Subscribe topic error, Parameter Error") raise ActionError(KafkaErr.ParameterError)
def create_topic(self, topic_name: str): """ 在集群中创建新的topic(topic配置采用默认模式) :param topic_name: :return: """ topic_list = [self.new_topic(topic_name)] try: response = self.admin_client.create_topics( topic_list, timeout_ms=TIME_OUT_ADMIN) except TopicAlreadyExistsError: log.tag_error( KafkaInfo.KafkaAdmin, "Topic [%s] already exist! Create Failed !" % topic_name) raise ActionError(KafkaErr.TopicExist) return response
def end_offsets(self, partitions: list): """ 获取指定partition结束的offset :param partitions: 指定topic分区[(topic: partition)] :return: """ _partitions = [TopicPartition(_par[0], _par[1]) for _par in partitions] try: result = self.consumer.end_offsets(_partitions) except UnsupportedVersionError or KafkaTimeoutError as e: if e.__class__ == UnsupportedVersionError: log.tag_error(KafkaInfo.KafkaConsumer, "API VERSION ERROR, DO NOT SUPPORT") raise ActionError(KafkaErr.NotSupport) else: log.tag_error(KafkaInfo.KafkaConsumer, "Get end offset failed, Time out") raise ActionError(KafkaErr.GetOffsetFailed) return result
def offsets_for_time(self, partitions_time: list, timestamp: int = -1): """ 寻找指定时间后的partition最早offset :param partitions_time: list of (topic, partition) if timestamp > 0, (topic, partition, timestamp) if timestamp = -1 :param timestamp: 指定的开始查询时间, 如果是-1则表示每个partitions都有自己的时间配置 :return: """ if timestamp > 0: _partitions = { TopicPartition(_tuple[0], _tuple[1]): timestamp for _tuple in partitions_time } else: _partitions = { TopicPartition(_tuple[0], _tuple[1]): _tuple[2] for _tuple in partitions_time } try: result = self.consumer.offsets_for_times(_partitions) except UnsupportedVersionError or ValueError or KafkaTimeoutError as e: if e.__class__ == UnsupportedVersionError: log.tag_error(KafkaInfo.KafkaConsumer, "API VERSION ERROR, DO NOT SUPPORT") raise ActionError(KafkaErr.NotSupport) if e.__class__ == ValueError: log.tag_error(KafkaInfo.KafkaConsumer, "Value Error: Target Timestamp is negative") else: log.tag_error(KafkaInfo.KafkaConsumer, "Get offset by timestamp failed, Time out") raise ActionError(KafkaErr.GetOffsetFailed) return result