def test_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id="test-kafka-feed") self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False)))) producer = KeyedProducer(get_kafka_client_or_none()) offsets = feed.get_current_offsets() send_to_kafka( producer, topics.FORM, ChangeMeta(document_id="1", data_source_type="form", data_source_name="form") ) send_to_kafka( producer, topics.CASE, ChangeMeta(document_id="2", data_source_type="case", data_source_name="case") ) send_to_kafka( producer, topics.FORM_SQL, ChangeMeta(document_id="3", data_source_type="form-sql", data_source_name="form-sql"), ) send_to_kafka( producer, topics.CASE_SQL, ChangeMeta(document_id="4", data_source_type="case-sql", data_source_name="case-sql"), ) changes = list(feed.iter_changes(since=offsets, forever=False)) self.assertEqual(2, len(changes)) self.assertEqual(set(["1", "2"]), set([change.id for change in changes]))
def get_default_couch_db_change_feed_pillow(): default_couch_db = CachedCouchDB(CommCareCase.get_db().uri, readonly=False) kafka_client = get_kafka_client_or_none() return ChangeFeedPillow( couch_db=default_couch_db, kafka=kafka_client, checkpoint=PillowCheckpoint(get_django_checkpoint_store(), 'default-couch-change-feed') )
def kafka(self): # load everything lazily to avoid doing this work if not needed if self._kafka is None and not self._has_error: self._kafka = get_kafka_client_or_none() if self._kafka is None: logging.warning('Kafka is not available! Change producer is doing nothing.') self._has_error = True return self._kafka
def kafka(self): # load everything lazily to avoid doing this work if not needed if self._kafka is None and not self._has_error: self._kafka = get_kafka_client_or_none() if self._kafka is None: _assert = soft_assert(notify_admins=True) _assert(settings.DEBUG, 'Kafka is not available! Change producer is doing nothing.') self._has_error = True return self._kafka
def check_kafka(): client = get_kafka_client_or_none() if not client: return ServiceStatus(False, "Could not connect to Kafka") elif len(client.brokers) == 0: return ServiceStatus(False, "No Kafka brokers found") elif len(client.topics) == 0: return ServiceStatus(False, "No Kafka topics found") else: return ServiceStatus(True, "Kafka seems to be in order")
def get_change_feed_pillow_for_db(pillow_id, couch_db): kafka_client = get_kafka_client_or_none() processor = KafkaProcessor( kafka_client, data_source_type=data_sources.COUCH, data_source_name=couch_db.dbname ) checkpoint = PillowCheckpoint(pillow_id) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=CouchChangeFeed(couch_db, include_docs=True), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_change_feed_pillow_for_db(pillow_id, couch_db): kafka_client = get_kafka_client_or_none() processor = KafkaProcessor( kafka_client, data_source_type=data_sources.COUCH, data_source_name=couch_db.dbname ) change_feed = CouchChangeFeed(couch_db, include_docs=True) checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_user_groups_db_kafka_pillow(pillow_id): # note: this is temporarily using ConstructedPillow as a test. If it is successful we should # flip the main one over as well user_groups_couch_db = couch_config.get_db_for_class(CommCareUser) kafka_client = get_kafka_client_or_none() processor = KafkaProcessor( kafka_client, data_source_type=data_sources.COUCH, data_source_name=user_groups_couch_db.dbname ) checkpoint = PillowCheckpoint(pillow_id) return ConstructedPillow( name=pillow_id, document_store=None, # because we're using include_docs we can be explicit about not using this checkpoint=checkpoint, change_feed=CouchChangeFeed(user_groups_couch_db, include_docs=True), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def _get_producer(): return SimpleProducer(get_kafka_client_or_none())
def check_kafka(): client = get_kafka_client_or_none() if not client: return ServiceStatus(False, "Could not connect to Kafka") # TODO elaborate? return ServiceStatus(True, "Kafka's fine. Probably.")
def _get_producer(): return KeyedProducer(get_kafka_client_or_none())