def __init__(self, url): """Init url - kafka connection details """ self._kafka = kafka_client.KafkaClient(url) self._producer = kafka_producer.KeyedProducer( self._kafka, async=False, req_acks=kafka_producer.KeyedProducer.ACK_AFTER_LOCAL_WRITE, ack_timeout=2000)
def __init__(self, kafka_url, zookeeper_url, zookeeper_path, group, topic, fetch_size=1048576, repartition_callback=None, commit_callback=None, commit_timeout=30, timeout=DEFAULT_SOCKET_TIMEOUT_SECONDS): """Init kafka_url - Kafka location zookeeper_url - Zookeeper location zookeeper_path - Zookeeper path used for partition negotiation group - Kafka consumer group topic - Kafka topic repartition_callback - Callback to run when the Kafka consumer group changes. Repartitioning takes a relatively long time so this is a good time to flush and commit any data. commit_callback - Callback to run when the commit_timeout has elapsed between commits. commit_timeout - Timeout between commits. timeout - kafka connection timeout """ self._kazoo_client = None self._set_partitioner = None self._repartition_callback = repartition_callback self._commit_callback = commit_callback self._commit_timeout = commit_timeout self._last_commit = 0 self._partitions = [] self._kafka_group = group self._kafka_topic = topic self._kafka_fetch_size = fetch_size self._zookeeper_url = zookeeper_url self._zookeeper_path = zookeeper_path self._kafka = kafka_client.KafkaClient(kafka_url, timeout=timeout) self._consumer = self._create_kafka_consumer()
def health_check(self): url = CONF.kafka.uri try: kafka_client = client.KafkaClient(hosts=url) except client.KafkaUnavailableError as ex: LOG.error(repr(ex)) error_str = 'Could not connect to Kafka at {0}'.format(url) return base.CheckResult(healthy=False, message=error_str) status = self._verify_topics(kafka_client) self._disconnect_gracefully(kafka_client) return base.CheckResult(healthy=status[0], message=status[1])
def healthcheck(self): url = CONF.kafka_healthcheck.kafka_url try: kafka_client = client.KafkaClient(hosts=url) except client.KafkaUnavailableError as ex: LOG.error(repr(ex)) error_str = 'Could not connect to kafka at %s' % url return CheckResult(healthy=False, message=error_str) result = self._verify_topics(kafka_client) self._disconnect_gracefully(kafka_client) return result
def __enter__(self): self.kafka_conn = client.KafkaClient(self.connect_str) return self.kafka_conn