def test_process_change(self): consumer = KafkaConsumer( topics.CASE, group_id='test-consumer', bootstrap_servers=[settings.KAFKA_URL], consumer_timeout_ms=100, ) pillow = ChangeFeedPillow(self._fake_couch, kafka=get_kafka_client(), checkpoint=None) document = { 'doc_type': 'CommCareCase', 'type': 'mother', 'domain': 'kafka-test-domain', } pillow.process_change(Change(id='test-id', sequence_id='3', document=document)) message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(COUCH, change_meta.data_source_type) self.assertEqual(self._fake_couch.dbname, change_meta.data_source_name) self.assertEqual('test-id', change_meta.document_id) self.assertEqual(document['doc_type'], change_meta.document_type) self.assertEqual(document['type'], change_meta.document_subtype) self.assertEqual(document['domain'], change_meta.domain) self.assertEqual(False, change_meta.is_deletion) with self.assertRaises(ConsumerTimeout): consumer.next()
def _get_topic_offsets(topics, latest): """ :param topics: list of topics :param latest: True to fetch latest offsets, False to fetch earliest available :return: dict: { (topic, partition): offset, ... } """ # https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetRequest # https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/ assert set(topics) <= set(ALL) client = get_kafka_client() partition_meta = client.topic_partitions # only return the offset of the latest message in the partition num_offsets = 1 time_value = -1 if latest else -2 offsets = {} offset_requests = [] for topic in topics: partitions = list(partition_meta.get(topic, {})) for partition in partitions: offsets[(kafka_bytestring(topic), partition)] = None offset_requests.append( OffsetRequest(kafka_bytestring(topic), partition, time_value, num_offsets)) responses = client.send_offset_request(offset_requests) for r in responses: offsets[(kafka_bytestring(r.topic), r.partition)] = r.offsets[0] return offsets
def create_kafka_topics(): client = get_kafka_client() for topic in topics.ALL: if client.has_metadata_for_topic(topic): status = "already exists" else: client.ensure_topic_exists(topic, timeout=10) status = "created" print("topic {}: {}".format(status, topic))
def create_kafka_topics(): client = get_kafka_client() for topic in topics.ALL: if client.has_metadata_for_topic(topic): status = "already exists" else: client.ensure_topic_exists(topic, timeout=10) status = "created" print("topic {}: {}".format(status, topic))
def _get_topic_offsets(topics, latest): # https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetRequest time_value = -1 if latest else -2 assert set(topics) <= set(ALL) client = get_kafka_client() offset_requests = [OffsetRequest(topic, 0, time_value, 1) for topic in topics] responses = client.send_offset_request(offset_requests) return { r.topic: r.offsets[0] for r in responses }
def _get_topic_offsets(topics, latest): # https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetRequest time_value = -1 if latest else -2 assert set(topics) <= set(ALL) client = get_kafka_client() offset_requests = [ OffsetRequest(topic, 0, time_value, 1) for topic in topics ] responses = client.send_offset_request(offset_requests) return {r.topic: r.offsets[0] for r in responses}
def setUp(cls): cls._fake_couch = FakeCouchDb() cls._fake_couch.dbname = 'test-couchdb' with trap_extra_setup(KafkaUnavailableError): cls.consumer = KafkaConsumer( topics.CASE, group_id='test-consumer', bootstrap_servers=[settings.KAFKA_URL], consumer_timeout_ms=100, ) cls.pillow = ChangeFeedPillow(cls._fake_couch, kafka=get_kafka_client(), checkpoint=None)
def check_kafka(): try: client = get_kafka_client() except Exception as e: return ServiceStatus(False, "Could not connect to Kafka: %s" % e) if len(client.cluster.brokers()) == 0: return ServiceStatus(False, "No Kafka brokers found") elif len(client.cluster.topics()) == 0: return ServiceStatus(False, "No Kafka topics found") else: return ServiceStatus(True, "Kafka seems to be in order")
def check_kafka(): try: client = get_kafka_client() except Exception as e: return ServiceStatus(False, "Could not connect to Kafka: %s" % e) if len(client.cluster.brokers()) == 0: return ServiceStatus(False, "No Kafka brokers found") elif len(client.cluster.topics()) == 0: return ServiceStatus(False, "No Kafka topics found") else: return ServiceStatus(True, "Kafka seems to be in order")