class KafkaService():
LOGGER = logging.getLogger(__name__)
KAFKA_TOPIC = 'world2kafka'
BOOTSTRAP_SERVER = 'localhost:9092'
_producer = _KafkaProducer(bootstrap_servers=BOOTSTRAP_SERVER)
@staticmethod
def publish(message: str):
future = KafkaService._producer.send(KafkaService.KAFKA_TOPIC,
bytes(message.encode('utf-8')))
try:
record_metadata = future.get(timeout=10)
KafkaService.LOGGER.info(
"Published to Kafka: {}".format(record_metadata))
return record_metadata
except KafkaError as e:
KafkaService.LOGGER.error("Error attempting to publish message.",
e)
pass
@staticmethod
def subscribe_from_start():
return _KafkaConsumer(
KafkaService.KAFKA_TOPIC,
bootstrap_servers=[KafkaService.BOOTSTRAP_SERVER],
auto_offset_reset='earliest',
enable_auto_commit=True)
def __init__(self, bootstrap_servers: str, topic: str):
"""
Create the notifier.
:param bootstrap_servers: the Kafka bootstrap servers parameter.
:param topic: the topic where messages will be sent. The notifier requires the topic
name to consist of ASCII alphanumeric values and the hyphen to avoid Kafka issues
around ambiguity between period and underscore values.
"""
_check_string(bootstrap_servers, 'bootstrap_servers')
self._topic = _check_string(topic, 'topic', max_len=249)
match = self._KAFKA_TOPIC_ILLEGAL_CHARS_RE.search(
_cast(str, self._topic))
if match:
raise ValueError(
f'Illegal character in Kafka topic {self._topic}: {match.group()}'
)
# TODO LATER KAFKA support delivery.timeout.ms when the client supports it
# https://github.com/dpkp/kafka-python/issues/1723
# since not supported, we half ass it with a retry count.
# See https://kafka.apache.org/documentation/#producerconfigs
# this will fail if it can't connect
self._prod = _KafkaProducer(
# can't test multiple servers without a massive PITA
bootstrap_servers=bootstrap_servers.split(','),
acks='all',
# retries can occur from 100-30000 ms by default. If we allow 300 retries, that means
# the send can take from 30s to 150m. Presumably another server timeout will kill
# the request before then.
retries=300,
retry_backoff_ms=100, # default is 100
# low timeouts can cause message loss, apparently:
# https://github.com/dpkp/kafka-python/issues/1723
request_timeout_ms=30000, # default is 30000
# presumably this can be removed once idempotence is supported
max_in_flight_requests_per_connection=1,
)
self._closed = False
def __init__(self, **configs):
super(KafkaProducer, self).__init__(**configs)
self._producer = _KafkaProducer(
value_serializer=pydantic_to_json_serializer, **configs)
def __init__(self, server, topic, msg_type):
super().__init__()
self._producer = _KafkaProducer(bootstrap_servers=server)
self._topic = topic
self._msg_type = msg_type
def __init__(self, broker):
self.producer = _KafkaProducer(bootstrap_servers=[broker],
value_serializer=lambda eml: json.dumps(eml.dumps()).encode('utf-8'))