def __init__(self, url):
"""Init
url - kafka connection details
"""
self._kafka = kafka_client.KafkaClient(url)
self._producer = kafka_producer.KeyedProducer(
self._kafka,
async=False,
req_acks=kafka_producer.KeyedProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=2000)
def __init__(self,
kafka_url,
zookeeper_url,
zookeeper_path,
group,
topic,
fetch_size=1048576,
repartition_callback=None,
commit_callback=None,
commit_timeout=30,
timeout=DEFAULT_SOCKET_TIMEOUT_SECONDS):
"""Init
kafka_url - Kafka location
zookeeper_url - Zookeeper location
zookeeper_path - Zookeeper path used for partition
negotiation
group - Kafka consumer group
topic - Kafka topic
repartition_callback - Callback to run when the Kafka consumer
group changes. Repartitioning takes a
relatively long time so this is a good
time to flush and commit any data.
commit_callback - Callback to run when the commit_timeout
has elapsed between commits.
commit_timeout - Timeout between commits.
timeout - kafka connection timeout
"""
self._kazoo_client = None
self._set_partitioner = None
self._repartition_callback = repartition_callback
self._commit_callback = commit_callback
self._commit_timeout = commit_timeout
self._last_commit = 0
self._partitions = []
self._kafka_group = group
self._kafka_topic = topic
self._kafka_fetch_size = fetch_size
self._zookeeper_url = zookeeper_url
self._zookeeper_path = zookeeper_path
self._kafka = kafka_client.KafkaClient(kafka_url, timeout=timeout)
self._consumer = self._create_kafka_consumer()
def health_check(self):
url = CONF.kafka.uri
try:
kafka_client = client.KafkaClient(hosts=url)
except client.KafkaUnavailableError as ex:
LOG.error(repr(ex))
error_str = 'Could not connect to Kafka at {0}'.format(url)
return base.CheckResult(healthy=False, message=error_str)
status = self._verify_topics(kafka_client)
self._disconnect_gracefully(kafka_client)
return base.CheckResult(healthy=status[0], message=status[1])
def healthcheck(self):
url = CONF.kafka_healthcheck.kafka_url
try:
kafka_client = client.KafkaClient(hosts=url)
except client.KafkaUnavailableError as ex:
LOG.error(repr(ex))
error_str = 'Could not connect to kafka at %s' % url
return CheckResult(healthy=False, message=error_str)
result = self._verify_topics(kafka_client)
self._disconnect_gracefully(kafka_client)
return result
def __enter__(self):
self.kafka_conn = client.KafkaClient(self.connect_str)
return self.kafka_conn