def test_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id="test-kafka-feed")
self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False))))
producer = KeyedProducer(get_kafka_client_or_none())
offsets = feed.get_current_offsets()
send_to_kafka(
producer, topics.FORM, ChangeMeta(document_id="1", data_source_type="form", data_source_name="form")
)
send_to_kafka(
producer, topics.CASE, ChangeMeta(document_id="2", data_source_type="case", data_source_name="case")
)
send_to_kafka(
producer,
topics.FORM_SQL,
ChangeMeta(document_id="3", data_source_type="form-sql", data_source_name="form-sql"),
)
send_to_kafka(
producer,
topics.CASE_SQL,
ChangeMeta(document_id="4", data_source_type="case-sql", data_source_name="case-sql"),
)
changes = list(feed.iter_changes(since=offsets, forever=False))
self.assertEqual(2, len(changes))
self.assertEqual(set(["1", "2"]), set([change.id for change in changes]))
def get_default_couch_db_change_feed_pillow():
default_couch_db = CachedCouchDB(CommCareCase.get_db().uri, readonly=False)
kafka_client = get_kafka_client_or_none()
return ChangeFeedPillow(
couch_db=default_couch_db,
kafka=kafka_client,
checkpoint=PillowCheckpoint(get_django_checkpoint_store(), 'default-couch-change-feed')
)
def kafka(self):
# load everything lazily to avoid doing this work if not needed
if self._kafka is None and not self._has_error:
self._kafka = get_kafka_client_or_none()
if self._kafka is None:
logging.warning('Kafka is not available! Change producer is doing nothing.')
self._has_error = True
return self._kafka
def kafka(self):
# load everything lazily to avoid doing this work if not needed
if self._kafka is None and not self._has_error:
self._kafka = get_kafka_client_or_none()
if self._kafka is None:
_assert = soft_assert(notify_admins=True)
_assert(settings.DEBUG, 'Kafka is not available! Change producer is doing nothing.')
self._has_error = True
return self._kafka
def kafka(self):
# load everything lazily to avoid doing this work if not needed
if self._kafka is None and not self._has_error:
self._kafka = get_kafka_client_or_none()
if self._kafka is None:
_assert = soft_assert(notify_admins=True)
_assert(settings.DEBUG, 'Kafka is not available! Change producer is doing nothing.')
self._has_error = True
return self._kafka
def check_kafka():
client = get_kafka_client_or_none()
if not client:
return ServiceStatus(False, "Could not connect to Kafka")
elif len(client.brokers) == 0:
return ServiceStatus(False, "No Kafka brokers found")
elif len(client.topics) == 0:
return ServiceStatus(False, "No Kafka topics found")
else:
return ServiceStatus(True, "Kafka seems to be in order")
def check_kafka():
client = get_kafka_client_or_none()
if not client:
return ServiceStatus(False, "Could not connect to Kafka")
elif len(client.brokers) == 0:
return ServiceStatus(False, "No Kafka brokers found")
elif len(client.topics) == 0:
return ServiceStatus(False, "No Kafka topics found")
else:
return ServiceStatus(True, "Kafka seems to be in order")
def get_change_feed_pillow_for_db(pillow_id, couch_db):
kafka_client = get_kafka_client_or_none()
processor = KafkaProcessor(
kafka_client, data_source_type=data_sources.COUCH, data_source_name=couch_db.dbname
)
checkpoint = PillowCheckpoint(pillow_id)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=CouchChangeFeed(couch_db, include_docs=True),
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_change_feed_pillow_for_db(pillow_id, couch_db):
kafka_client = get_kafka_client_or_none()
processor = KafkaProcessor(
kafka_client, data_source_type=data_sources.COUCH, data_source_name=couch_db.dbname
)
change_feed = CouchChangeFeed(couch_db, include_docs=True)
checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_user_groups_db_kafka_pillow(pillow_id):
# note: this is temporarily using ConstructedPillow as a test. If it is successful we should
# flip the main one over as well
user_groups_couch_db = couch_config.get_db_for_class(CommCareUser)
kafka_client = get_kafka_client_or_none()
processor = KafkaProcessor(
kafka_client, data_source_type=data_sources.COUCH, data_source_name=user_groups_couch_db.dbname
)
checkpoint = PillowCheckpoint(pillow_id)
return ConstructedPillow(
name=pillow_id,
document_store=None, # because we're using include_docs we can be explicit about not using this
checkpoint=checkpoint,
change_feed=CouchChangeFeed(user_groups_couch_db, include_docs=True),
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def _get_producer():
return SimpleProducer(get_kafka_client_or_none())
def check_kafka():
client = get_kafka_client_or_none()
if not client:
return ServiceStatus(False, "Could not connect to Kafka")
# TODO elaborate?
return ServiceStatus(True, "Kafka's fine. Probably.")
def check_kafka():
client = get_kafka_client_or_none()
if not client:
return ServiceStatus(False, "Could not connect to Kafka")
# TODO elaborate?
return ServiceStatus(True, "Kafka's fine. Probably.")
def _get_producer():
return KeyedProducer(get_kafka_client_or_none())
def _get_producer():
return KeyedProducer(get_kafka_client_or_none())
