def test_checkpoint_with_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed')
pillow_name = 'test-multi-topic-checkpoints'
checkpoint = PillowCheckpoint(pillow_name)
processor = CountingProcessor()
pillow = ConstructedPillow(
name=pillow_name,
checkpoint=checkpoint,
change_feed=feed,
processor=processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed
)
)
offsets = feed.get_latest_offsets()
self.assertEqual(set([topics.FORM, topics.CASE]), set(offsets.keys()))
# send a few changes to kafka so they should be picked up by the pillow
publish_stub_change(topics.FORM)
publish_stub_change(topics.FORM)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE_SQL)
pillow.process_changes(since=offsets, forever=False)
self.assertEqual(4, processor.count)
self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
publish_stub_change(topics.FORM)
publish_stub_change(topics.FORM)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE_SQL)
pillow.process_changes(pillow.get_last_checkpoint_sequence(), forever=False)
self.assertEqual(8, processor.count)
self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
def __init__(self, processor, pillow_name):
change_feed = KafkaChangeFeed(topics.ALL, group_id=pillow_name)
checkpoint = PillowCheckpoint(pillow_name)
event_handler = MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=1000, change_feed=change_feed
)
super(ConfigurableReportKafkaPillow, self).__init__(
name=pillow_name,
change_feed=change_feed,
processor=processor,
checkpoint=checkpoint,
change_processed_event_handler=event_handler
)
# set by the superclass constructor
assert self._processor is not None
assert self._processor.bootstrapped is not None
def get_case_to_elasticsearch_pillow(pillow_id='CaseToElasticsearchPillow'):
assert pillow_id == 'CaseToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, CASE_INDEX_INFO)
case_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_prep_fn=transform_case_for_elasticsearch
)
kafka_change_feed = KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='cases-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=case_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed
),
)
def get_unknown_users_pillow(pillow_id='unknown-users-pillow'):
"""
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, USER_INDEX_INFO)
processor = UnknownUsersProcessor()
change_feed = KafkaChangeFeed(topics=[FORM, FORM_SQL],
group_id='unknown-users')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_xform_to_elasticsearch_pillow(pillow_id='XFormToElasticsearchPillow'):
assert pillow_id == 'XFormToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, XFORM_INDEX_INFO)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_prep_fn=transform_xform_for_elasticsearch,
doc_filter_fn=xform_pillow_filter,
)
kafka_change_feed = KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='forms-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed
),
)
def get_user_pillow(pillow_id='UserPillow'):
assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, USER_INDEX_INFO)
user_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_prep_fn=transform_user_for_elasticsearch,
)
change_feed = KafkaChangeFeed(topics=[COMMCARE_USER, WEB_USER],
group_id='users-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=user_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_form_submission_metadata_tracker_pillow(
pillow_id='FormSubmissionMetadataTrackerProcessor'):
"""
This gets a pillow which iterates through all forms and marks the corresponding app
as having submissions. This could be expanded to be more generic and include
other processing that needs to happen on each form
"""
checkpoint = PillowCheckpoint('form-submission-metadata-tracker')
form_processor = FormSubmissionMetadataTrackerProcessor()
change_feed = KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL],
group_id='form-processsor')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=form_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed,
),
)