def _consume(self, executor):
if not executor:
raise KafkaException(KafkaCodeEnum.KFK_0009, "data executor null")
records = None
while self.__loop and self.__lock.acquire():
duration = self.get_convertor().duration_value()
try:
records = self._get_consumer().poll(duration)
except AbstractException as e:
if self.__loop:
if self.get_convertor().consumer_ignore_value().consume_throwable():
msg = "pool data:{} {}"
raise KafkaException(KafkaCodeEnum.KFK_0010, msg.format(duration, e.get_message()))
logger.error("{} pool data:{}".format(self.__loop, e.get_message()))
finally:
self.__lock.release()
if not records or not self.__loop:
continue
self._processor.consume_ready(self._get_consumer(), executor)
try:
# Python版本修改功能：由于数据结构不同
data = []
[data.extend(i) for i in records.values()]
#
for item in data:
self._processor.consume(self._get_consumer(), item, executor)
self._processor.consume_complete(self._get_consumer(), executor)
except KafkaException as e:
self._processor.consume_complete(self._get_consumer(), executor, e)
logger.error("consume data:{}".format(e.get_message()))
raise e
def assign_config(self):
config = self.__get_config_nested("topicPartition", ())
if not config:
raise KafkaException(KafkaCodeEnum.KFK_0002,
"config 'topicPartition' null")
tp = []
for item in config:
partitions = str(item).split("#")
if len(partitions) < 2:
raise KafkaException(KafkaCodeEnum.KFK_0003,
"topic partition format:T#[NUM]#[NUM]")
for i, partition in enumerate(partitions):
if i == 0:
continue
partition = NumberUtils.to_int(partition, -1)
if partition < 0:
raise KafkaException(KafkaCodeEnum.KFK_0003,
"partition format:[NUM]")
tp.append(TopicPartition(partitions[0], partition))
# tp.append({"topic": partitions[0], "partition": partition})
return tp
def __offset_commit(self, consumer, force):
# {TopicPartition: OffsetAndMetadata}
tmp = self.get_offset_cache()
if self.__mode.is_sync() or force:
try:
if self.__mode.is_each() and not force:
if tmp:
consumer.commit(tmp)
else:
consumer.commit()
except Exception as e:
if self.__ignore.offset_throwable():
raise KafkaException(
KafkaCodeEnum.KFK_0006,
"consumer commit:{} {}".format(tmp, str(e)))
else:
logger.warning("consumer commit:{} {}".format(tmp, str(e)))
return
else:
self.__make_committed(tmp)
elif self.__mode.is_each():
consumer.commit_async(
tmp, lambda offset, set_e: self._callback(offset, set_e))
else:
consumer.commit_async(
callback=lambda offset, set_e: self._callback(offset, set_e))
logger.debug("consumer commit:{}".format(tmp))
def topic_pattern_config(self):
value = self.__get_config_nested("topicPattern", "")
if not value:
raise KafkaException(KafkaCodeEnum.KFK_0004,
"config 'topicPattern' null")
return value
def __parse(self, config):
if not isinstance(config, dict) and len(config) == 0:
raise KafkaException(BasicCodeEnum.MSG_0005, "base config invalid")
client_c = config.get("client", {})
client_c = client_c if client_c else {}
config_c = config.get("config", {})
config_c = config_c if config_c else {}
self.__template = config.get("template", "")
self.__try_use_template(client_c, config_c)
self.__config = {"client": client_c, "config": config_c}
def accept(self, bridge, ignore):
try:
bridge.passing()
logger.debug("consume:{}".format(bridge.get_id()))
except KafkaException as e:
raise e
except Exception as e:
if ignore.consume_throwable():
msg = "consume:{} {}".format(bridge.get_id(), str(e))
raise KafkaException(KafkaCodeEnum.KFK_0010, msg)
else:
logger.warning("consume:{} {}".format(bridge.get_id(), str(e)))
def sent_data(self, producer, record, callback):
try:
# Python版本修改功能：由于方法参数不一致
tmp = producer.send(**record).add_callback(
lambda metadata: self._on_completion(metadata, callback)
).add_errback(lambda set_e: self._on_error(set_e, callback))
if self.__mode.is_sync():
tmp.get()
return tmp
except Exception as e:
raise KafkaException(KafkaCodeEnum.KFK_0012,
"data send:{}".format(str(e)))
def set(self, mode):
if not hasattr(ConsumerModeEnum, str(mode)):
raise KafkaException(KafkaCodeEnum.KFK_0001,
"adapter not support:{}".format(mode))
self.__mode = mode
def initialize(self, config):
if self.__initialized:
raise KafkaException(BasicCodeEnum.MSG_0020, "client initialized")
else:
self.__initialized = True
self.__convertor = ConfigConvertor(config)
def from_fn(cls, mode):
mode = str(mode).upper()
if not hasattr(cls, mode):
raise KafkaException(KafkaCodeEnum.KFK_0011, mode)
return getattr(cls, mode)
def abort_transaction(self, producer):
if self.__mode.is_transaction():
# Python版本不支持producer.abort_transaction方法
raise KafkaException(
KafkaCodeEnum.KFK_0015,
"{} not supported 'abort_transaction'".format(__version__))
def commit_transaction(self, producer, offsets, group_id):
if self.__mode.is_transaction():
# Python版本不支持producer.commit_transaction方法
raise KafkaException(
KafkaCodeEnum.KFK_0015,
"{} not supported 'commit_transaction'".format(__version__))
def _get_producer(self):
if not self.__producer:
raise KafkaException(KafkaCodeEnum.KFK_0005, "producer not initialize")
return self.__producer
def _get_consumer(self):
if not self.__consumer:
raise KafkaException(KafkaCodeEnum.KFK_0010, "consumer not initialize")
return self.__consumer