def error_cb(self, err):
print('error_cb --------> {}'.format(err))
if err.code() == KafkaError._ALL_BROKERS_DOWN:
raise KafkaException('ERROR: all brokers down...')
else:
print(err.code())
raise KafkaException(err.code())
def base_callback(self, err, msg):
self._producer._log.debug(
"Entering base callback with err=%s and msg=%s.", str(err),
str(msg))
if err is not None:
self._producer._log.debug("Callback failed (non-None err).")
super().failure(KafkaException(err))
elif msg is None:
self._producer._log.debug("Callback failed (None msg).")
super().failure(KafkaException(KafkaError.UNKNOWN, 'null msg'))
elif msg.error() != None:
self._producer._log.debug("Callback failed (non-None msg.error).")
super().failure(KafkaException(msg.error()))
else:
# success
self._producer._log.debug("Callback success.")
metadata = RecordMetadata(
msg.topic(), msg.partition(),
TopicPartition(msg.topic(), msg.partition()), msg.offset(),
msg.timestamp()[1] if
msg.timestamp()[0] != TIMESTAMP_NOT_AVAILABLE else int(time() *
1000),
None, self.key_len, self.value_len, -1)
super().success(metadata)
self._producer._log.debug(
"Finished with base callback with err=%s and msg=%s.", str(err),
str(msg))
def _fail_fast_consume(err: Any, partitions: Any) -> None:
if err is not None:
print(f"Kafka consumer commit error: {err}", file=sys.stderr)
raise KafkaException(err)
for p in partitions:
# check for partition-specific commit errors
if p.error:
print(
f"Kafka consumer commit error: {p.error}",
file=sys.stderr,
)
raise KafkaException(p.error)
def fail_fast(err, partitions):
if err is not None:
print("Kafka consumer commit error: {}".format(err))
print("Bailing out...")
# TODO: should it be sys.exit(-1)?
raise KafkaException(err)
for p in partitions:
# check for partition-specific commit errors
if p.error:
print("Kafka consumer commit error: {}".format(p.error))
print("Bailing out...")
# TODO: should it be sys.exit(-1)?
raise KafkaException(err)
#print("Kafka consumer commit successful")
pass
def produce(self, topic, partition, key, value, timeout_s):
self.last_msg = None
self.producer.produce(topic,
key=key.encode('utf-8'),
value=value.encode('utf-8'),
partition=partition,
callback=lambda e, m: self.on_delivery(e, m))
self.producer.flush(timeout_s)
msg = self.last_msg
if msg == None:
raise KafkaException(KafkaError(KafkaError._MSG_TIMED_OUT))
if msg.error() != None:
raise KafkaException(msg.error())
assert msg.offset() != None
return msg.offset()
def _receive_event(self, timeout=1) -> typing.Optional[bytes]:
try:
logger.debug("Start polling the message from queue...")
msg = self.consumer.poll(timeout=timeout)
logger.debug("Pooling was ended, checking returned data...")
except Exception as e:
logger.warning(f"Error polling: {e}")
return None
if msg is None:
logger.info(
"Polling timed out after %s sec. No queue message was received.",
timeout)
return None
if msg.error():
# Error or event
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
logger.info(
f"{msg.topic()} [msg.partition()] reached end at offset {msg.offset()}"
)
else:
# Error
kafka_error = msg.error()
ke = KafkaException(kafka_error)
if kafka_error.code() in [
KafkaError._TIMED_OUT_QUEUE, KafkaError._TIMED_OUT
]:
# ordinary error messages when no message was received.
logger.warning(
f"[TIMED_OUT errors] no messages were consumed throughout timeout - {timeout} sec: {ke}"
)
elif kafka_error.code() == KafkaError._TRANSPORT:
# when 'transport' was broken. Usually it'll be renewed automatically, but not all the time.
logger.error(
f"[Broker transport failure] Calling to re-subscribe. Error: {ke}"
)
self.subscribe_to_topic(
) # This is not 100% needed but it's better to have it here for now.
else:
ke = KafkaException(kafka_error)
logger.error(f"Error occurred: {ke}")
return None
else:
# Proper message
return msg.value()
def on_rebalance(consumer: Any, partitions: List[Any]) -> None:
for p in partitions:
if p.error:
raise KafkaException(p.error)
print("Kafka partitions rebalanced: {} / {}".format(
consumer, partitions),
file=sys.stderr)
def consume(conf, topic, proto_name):
consumer = Consumer(conf)
running = True
cnt = 0
try:
consumer.subscribe([topic])
while running:
msg: Message = consumer.poll(timeout=1.0)
if msg is None: continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
sys.stderr.write('%% %s [%d] reached end at offset %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
raise KafkaException(msg.error())
else:
cnt += 1
if proto_name == "":
print(msg.value())
else:
decode_proto(msg, proto_name)
print("Consumed ", cnt)
finally:
# Close down consumer to commit final offsets.
consumer.close()
def consume_loop(consumerObj, topics):
try:
consumerObj.subscribe(topics)
while running:
msg = consumerObj.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
sys.stderr.write(
'%% %s [%d] reached end at offset %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
raise KafkaException(msg.error())
else:
consumerObj.commit(asynchronous=False)
# msg_process(msg)
print("Start processing message - ", str(msg), msg.key(),
msg.value())
finally:
# Close down consumer to commit final offsets.
consumerObj.close()
def read(self, timeout: int = 0) -> Dict[Tuple[str, int], List[Message]]:
msg_count = 0
msg_dict: Dict[Tuple[str, int], List[Any]] = defaultdict(list)
try:
counter = 0
start_time = datetime.now()
while counter < 2:
msg = self.consumer.poll(timeout=5.0)
if msg is None:
counter += 1
continue
counter = 0
if msg.error():
raise KafkaException(msg.error())
message = Message(msg.offset(), msg.key(), msg.value(),
msg.timestamp()[1], msg.headers())
msg_dict[(msg.topic(), msg.partition())].append(message)
msg_count += 1
if not msg_count % 100000:
logger.debug(
f"So far read {msg_count} messages from kafka")
if datetime.now() - start_time > timedelta(seconds=timeout):
logger.info(
f'Reached timeout: {timeout}s for reading messages.')
break
logger.info("Done with reading")
except KeyboardInterrupt:
logger.info("%% Aborted by user\n")
finally:
self.consumer.close()
return msg_dict
def run(self):
LOG.info(f"{self.prefix} - Processor is running. Awaiting msgs.")
for msg in iter(self):
if msg.error():
LOG.error("%s - Consumer error: %s", self.prefix, msg.error())
kafka_failures.labels(reporter=self.reporter).inc()
raise KafkaException(msg.error())
try:
msg = json.loads(msg.value().decode("utf-8"))
self.handle_msg(msg)
except json.decoder.JSONDecodeError:
kafka_failures.labels(reporter=self.reporter).inc()
LOG.error(
'Unable to decode kafka message: %s - %s',
msg.value(), self.prefix
)
except Exception as err:
processor_requests_failures.labels(
reporter=self.reporter,
account_number=msg['input']['host']['account']
).inc()
LOG.error(
'An error occurred during message processing: %s - %s',
repr(err),
self.prefix
)
finally:
self.consumer.commit()
def fail_fast(err, msg):
if err is not None:
print("Kafka producer delivery error: {}".format(err),
file=sys.stderr)
print("Bailing out...", file=sys.stderr)
# TODO: should it be sys.exit(-1)?
raise KafkaException(err)
def basic_consume_loop(consumer, topics):
try:
consumer.subscribe(topics)
while running:
msg = consumer.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
sys.stderr.write(
'%% %s [%d] reached end at offset %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
raise KafkaException(msg.error())
else:
# msg_process(msg)
# print(dir(msg))
print(msg.value())
print(msg.partition())
finally:
# Close down consumer to commit final offsets.
consumer.close()
def read_summary(self):
print('SummaryDataReader is ready to consume the data from %s' % self.summary_topic_name)
self.kafka_client.subscribe([self.summary_topic_name])
try:
while True:
msg = self.kafka_client.consumer.poll(1.0)
if msg is None:
continue
if msg.error():
# Error or event
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
print('%% %s [%d] reached end at offset %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
# Error
raise KafkaException(msg.error())
else:
# Proper message
print('%% %s [%d] at offset %d with key %s:\n' %
(msg.topic(), msg.partition(), msg.offset(),
str(msg.key())))
print(msg.value())
except Exception as err:
logging.error(err)
# Close down consumer to commit final offsets.
finally:
self.kafka_client.close_consumer()
def _consume(group_id, topic, n, max_messages):
config = {
"bootstrap.servers": "localhost:9094",
"group.id": group_id,
"auto.offset.reset": "beginning",
"enable.partition.eof": "true",
"enable.auto.commit": "false",
}
consumer = Consumer(config)
consumer.subscribe(topics=[topic])
messages = 0
while True:
if messages == max_messages:
return
msg = consumer.consume(num_messages=n, timeout=5)
if len(msg) == 0:
continue
for m in msg:
if m.error():
if m.error().code() == KafkaError._PARTITION_EOF:
return
elif m.error():
raise KafkaException(m.error())
else:
messages += 1
if messages == max_messages:
break
consumer.commit(asynchronous=False)
def consume(self):
consumer_kafka_conf = {
"bootstrap.servers": self.bootstrap_servers,
'group.id': self.group_id,
'auto.offset.reset': 'smallest'
}
consumer = Consumer(consumer_kafka_conf)
try:
consumer.assign(
[TopicPartition(topic=self.topic, partition=self.partition)])
while self.stop is False:
msg = consumer.poll(timeout=1.0)
if msg is None:
sleep(0.2)
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
sys.stderr.write(
'%% %s [%d] reached end at offset %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
raise KafkaException(msg.error())
else:
self.model.train(msg.value().strip(), self.group_id)
if self.model.count == self.end:
self.stop = True
finally:
# Close down consumer to commit final offsets.
consumer.close()
def main() -> None:
from notifications.worker import execute_message
consumer = Consumer(
{
"bootstrap.servers": "kafka:29092",
"group.id": "notifications",
"auto.offset.reset": "earliest",
}
)
consumer.subscribe(["notifications"])
try:
while True:
msg = consumer.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
raise KafkaException(msg.error())
else:
try:
data = json.loads(msg.value())
execute_message(data)
except Exception as e:
print(msg.value())
print("Failed to handle message " + str(e))
except KeyboardInterrupt:
sys.stderr.write("%% Aborted by user\n")
finally:
# Close down consumer to commit final offsets.
consumer.close()
def read(self) -> Iterator[Message]:
# sometimes it is better to make constant call to KAFKA
# to keep the connection alive.
timedout = False
last_message_time = time.time()
while not timedout:
msg = self.consumer.poll(timeout=self.settings.poll_timeout)
if msg:
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
self.logger.warning("Reaching EOF")
break
else:
raise KafkaException(msg.error())
else:
self.logger.info("Read {}, {}".format(
msg.topic(), msg.offset()))
self.last_msg = msg
yield Message.deserialize(msg.value())
last_message_time = time.time()
if self.settings.timeout > 0:
time_since_last_message = time.time() - last_message_time
timedout = time_since_last_message > self.settings.timeout
def run(self):
global producer_total, consumer_total, consumer_topics, consumer_timeout
self.consumer.subscribe(consumer_topics)
msg = None
while self.runflag:
try:
msg = self.consumer.poll(timeout=consumer_timeout)
if msg is None:
raise AttributeError
if msg.error():
raise KafkaException(msg.error())
consumer_total += 1
value = msg.value().decode('utf-8')
socketio.emit('newMessage', {'msg': value},
namespace='/consumer')
print("-> consumer recv: ", value)
smsg = "EPD Simulator Totals --> Produced: {} Consumed: {}".format(
producer_total, consumer_total)
socketio.emit('newMessage', {'msg': smsg}, namespace='/epd')
#print(msg.key().decode('utf-8'), msg.value().decode('utf-8'))
gevent.sleep(0)
except KafkaException:
gevent.sleep(0.5)
if msg.error().code() != KafkaError._PARTITION_EOF:
print("KafkaException: ", msg.error())
# else:
# print("KafkaException: ", msg.error())
except AttributeError:
gevent.sleep(0.5)
def consume(self) -> Any:
"""
Consume messages from a give list of topic
:return:
"""
records = []
start = datetime.now()
try:
while True:
msg = self.consumer.poll(timeout=self.consumer_poll_timeout)
end = datetime.now()
# The consumer exceeds consume timeout
if (end - start) > timedelta(
seconds=self.consumer_total_timeout):
# Exceed the consume timeout
break
if msg is None:
continue
if msg.error():
# Hit the EOF of partition
if msg.error().code() == KafkaError._PARTITION_EOF:
continue
else:
raise KafkaException(msg.error())
else:
records.append(msg.value())
except Exception as e:
LOGGER.exception(e)
finally:
return records
def consume(self):
try:
while True:
msg = self.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
# Error or event
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
sys.stderr.write(
'%% %s [%d] reached end at offset %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
# Error
raise KafkaException(msg.error())
else:
# Proper message
sys.stderr.write('%% %s [%d] at offset %d with key %s:\n' %
(msg.topic(), msg.partition(),
msg.offset(), str(msg.key())))
print(msg.value())
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
# Close down consumer to commit final offsets.
self.close()
def receive_event(self, timeout=5) -> typing.Optional[bytes]:
msg = None
try:
msg = self.consumer.poll(timeout=timeout)
except Exception as e:
logger.warning("Error polling: {e}")
return None
if msg is None:
logger.warning(f"Polling timed out after {timeout} sec")
return None
if msg.error():
# Error or event
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
logger.info(
f"{msg.topic()} [msg.partition()] reached end at offset {msg.offset()}"
)
else:
# Error
ke = KafkaException(msg.error())
# raise e
logger.error(f"Error occurred: {ke}")
return None
else:
# Proper message
return msg.value()
def test_submit_upstream_alert_failure(self):
"""Test that a KafkaException results in an AlertSubmissionException."""
t = Target.objects.create(name='test name', ra=1, dec=2)
with patch('tom_scimma.scimma.Stream.open', mock_open(read_data='data')) as mock_stream:
mock_stream().write.side_effect = KafkaException()
with self.assertRaises(AlertSubmissionException):
SCIMMABroker().submit_upstream_alert(target=t, observation_record=None)
def consumeFromKafka(self, topic):
def print_assignment(consumer, partitions):
for partition in partitions:
print('Assignment:', partition)
# Subscribe to topic
self.consumer.subscribe(topic, on_assign=print_assignment)
try:
while True:
# Consume form topic
msg = self.consumer.poll(timeout=1.0)
if msg is None: continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
# End of partition event
pass
# sys.stderr.write('%s partition[%d] reached end at offset %d\n' %(msg.topic(), msg.partition(), msg.offset()))
elif msg.error():
raise KafkaException(msg.error())
else:
yield msg.value().decode('utf-8')
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user KeyboardInterrupt \n')
self.consumer.close()
def test_notify_sources_application_availability_kafka_exception(
self,
mock_kafka_producer,
):
"""Test notify source application availability handling KafkaException."""
kafka_producer = mock_kafka_producer(self.sources_kafka_config)
kafka_producer.produce.side_effect = KafkaException(KafkaError(5))
with override_settings(
LISTENER_SERVER=self.listener_server,
LISTENER_PORT=self.listener_port,
SOURCES_STATUS_TOPIC=self.sources_kafka_topic,
SOURCES_RESOURCE_TYPE=self.sources_resource_type,
SOURCES_AVAILABILITY_EVENT_TYPE=self.sources_availability_event_type,
SOURCES_ENABLE_DATA_MANAGEMENT_FROM_KAFKA=True,
):
with self.assertRaises(KafkaProducerException):
sources.notify_application_availability(
self.account_number,
self.application_id,
availability_status=self.available_status,
)
kafka_producer.produce.assert_called_with(
topic=self.sources_kafka_topic,
value=json.dumps(self.kafka_payload),
headers={
"x-rh-identity": self.headers["X-RH-IDENTITY"],
"event_type": self.sources_availability_event_type,
},
callback=_check_response,
)
kafka_producer.flush.assert_not_called()
async def read_stream(self):
# 监听kafka
try:
while True:
loop = asyncio.get_event_loop()
message = await loop.run_in_executor(None, self.consumer.poll)
if message is None:
continue
if message.error():
logger.exception(message.error())
raise KafkaException(message.error())
else:
data = json.loads(message.value().decode("utf-8"))
try:
finish = await self.handler(data)
except Exception as e:
logger.info(f"handler解析出现异常[{e}]")
time.sleep(1)
continue
if finish:
self.consumer.commit(asynchronus=True) # 不需要等待到触发回调函数之后.(?)
except Exception as e:
logger.error(f"kafka error.[{e}]")
return
def fetch_results(self, num=1):
self.establishCon(self.r_tpoic)
rs = []
attempts = 5
while (num > 0):
msg = self.consumer.poll(timeout=1.0)
if msg is None:
# stop consumer when reaching end
attempts -= 1
print('attemtps left %d', attempts)
if (attempts <= 0):
break
continue
if (msg and msg.error()):
print("raising error!")
raise KafkaException(msg.error())
else:
sys.stderr.write(
'%% Fetching Topic %s [%d] at offset %d with key %s:\n' %
(msg.topic(), msg.partition(), msg.offset(), str(
msg.key())))
# load from partitions, returned jd is a list
rs.append(json.loads(msg.value().decode('utf-8')))
num -= 1
# self.consumer.commit(async=False)
self.closeCon()
return rs
def test_consumer(self):
""" 消息消费者
"""
conf = {
'bootstrap.servers': self.broker,
'group.id': self.group_id,
'default.topic.config': {
'auto.offset.reset': 'smallest'
}
}
c = Consumer(conf)
c.subscribe(self.topic_con)
try:
while True:
msg = c.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
print(msg.topic(), msg.partition(), msg.offset())
elif msg.error():
raise KafkaException(msg.error())
else:
print('%% %s [%d] at offset %d with key %s:\n' %
(msg.topic(), msg.partition(), msg.offset(),
str(msg.key())))
print(msg.value())
except KeyboardInterrupt:
print('%% Aborted by user\n')
finally:
c.close()
def run(self):
try:
partition_ends = 0
total_parts, partitions = self._partitions()
self.consumer.assign(partitions)
while True:
msg = self.consumer.poll(timeout=0.5)
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
eprint(
f'{msg.topic()} reached end of partition [{msg.partition()}] at offset {msg.offset()}'
)
partition_ends += 1
if partition_ends == total_parts:
break
elif msg.error():
raise KafkaException(msg.error())
else:
record = {
'key': self._decode(self.key_decoder, msg.key()),
'payload': self._decode(self.value_decoder,
msg.value()),
'topic': msg.topic(),
'partition': msg.partition(),
'offset': msg.offset(),
'timestamp': msg.timestamp()[1]
}
print(json.dumps(record))
finally:
self.consumer.close()
def perform_consume(reset_offset_beginning: bool = False):
"""Perform consumer."""
conf = {
'bootstrap.servers': '127.0.0.1:9092',
'group.id': 'local-consumer',
'session.timeout.ms': 6000,
'auto.offset.reset': 'earliest'
}
topic_name = 'local.test.topic'
c = Consumer(conf)
if reset_offset_beginning:
c.subscribe([topic_name], on_assign=on_assign)
else:
c.subscribe([topic_name])
try:
while True:
msg = c.poll(timeout=1.0)
if msg is None:
print('Waiting for message or event/error in poll()')
elif msg.error():
raise KafkaException(msg.error())
else:
print(
f'{msg.topic()}[{msg.partition()}] at offset {msg.offset()}, key: {msg.key()}'
) # noqa
message = json.loads(msg.value())
print(message)
except KeyboardInterrupt:
print('aborted by user.')
finally:
c.close()
