def test_checkpoint_with_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed')
pillow_name = 'test-multi-topic-checkpoints'
checkpoint = PillowCheckpoint(pillow_name)
processor = CountingProcessor()
pillow = ConstructedPillow(
name=pillow_name,
document_store=None,
checkpoint=checkpoint,
change_feed=feed,
processor=processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed
)
)
offsets = feed.get_current_offsets()
self.assertEqual(set([topics.FORM, topics.CASE]), set(offsets.keys()))
# send a few changes to kafka so they should be picked up by the pillow
publish_stub_change(topics.FORM)
publish_stub_change(topics.FORM)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE_SQL)
pillow.process_changes(since=offsets, forever=False)
self.assertEqual(4, processor.count)
self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
publish_stub_change(topics.FORM)
publish_stub_change(topics.FORM)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE_SQL)
pillow.process_changes(pillow.get_last_checkpoint_sequence(), forever=False)
self.assertEqual(8, processor.count)
self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
def test_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id="test-kafka-feed")
self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False))))
producer = KeyedProducer(get_kafka_client_or_none())
offsets = feed.get_current_offsets()
send_to_kafka(
producer, topics.FORM, ChangeMeta(document_id="1", data_source_type="form", data_source_name="form")
)
send_to_kafka(
producer, topics.CASE, ChangeMeta(document_id="2", data_source_type="case", data_source_name="case")
)
send_to_kafka(
producer,
topics.FORM_SQL,
ChangeMeta(document_id="3", data_source_type="form-sql", data_source_name="form-sql"),
)
send_to_kafka(
producer,
topics.CASE_SQL,
ChangeMeta(document_id="4", data_source_type="case-sql", data_source_name="case-sql"),
)
changes = list(feed.iter_changes(since=offsets, forever=False))
self.assertEqual(2, len(changes))
self.assertEqual(set(["1", "2"]), set([change.id for change in changes]))
def test_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed')
self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False))))
offsets = feed.get_current_offsets()
expected_metas = [publish_stub_change(topics.FORM), publish_stub_change(topics.CASE)]
unexpected_metas = [publish_stub_change(topics.FORM_SQL), publish_stub_change(topics.CASE_SQL)]
changes = list(feed.iter_changes(since=offsets, forever=False))
self.assertEqual(2, len(changes))
found_change_ids = set([change.id for change in changes])
self.assertEqual(set([meta.document_id for meta in expected_metas]), found_change_ids)
for unexpected in unexpected_metas:
self.assertTrue(unexpected.document_id not in found_change_ids)
def test_multiple_topics_with_partial_checkpoint(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed')
self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False))))
offsets = {'form': feed.get_current_offsets()['form']}
expected_metas = [publish_stub_change(topics.FORM), publish_stub_change(topics.CASE)]
changes = list(feed.iter_changes(since=offsets, forever=False))
# should include at least the form and the case (may have more than one case since not
# specifying a checkpoint rewinds it to the beginning of the feed)
self.assertTrue(len(changes) > 1)
found_change_ids = set([change.id for change in changes])
for expected_id in set([meta.document_id for meta in expected_metas]):
self.assertTrue(expected_id in found_change_ids)
def test_multiple_topics_with_partial_checkpoint(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE],
group_id='test-kafka-feed')
self.assertEqual(
0, len(list(feed.iter_changes(since=None, forever=False))))
offsets = {'form': feed.get_current_offsets()['form']}
expected_metas = [
publish_stub_change(topics.FORM),
publish_stub_change(topics.CASE)
]
changes = list(feed.iter_changes(since=offsets, forever=False))
# should include at least the form and the case (may have more than one case since not
# specifying a checkpoint rewinds it to the beginning of the feed)
self.assertTrue(len(changes) > 1)
found_change_ids = set([change.id for change in changes])
for expected_id in set([meta.document_id for meta in expected_metas]):
self.assertTrue(expected_id in found_change_ids)
def test_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE],
group_id='test-kafka-feed')
self.assertEqual(
0, len(list(feed.iter_changes(since=None, forever=False))))
offsets = feed.get_current_offsets()
expected_metas = [
publish_stub_change(topics.FORM),
publish_stub_change(topics.CASE)
]
unexpected_metas = [
publish_stub_change(topics.FORM_SQL),
publish_stub_change(topics.CASE_SQL)
]
changes = list(feed.iter_changes(since=offsets, forever=False))
self.assertEqual(2, len(changes))
found_change_ids = set([change.id for change in changes])
self.assertEqual(set([meta.document_id for meta in expected_metas]),
found_change_ids)
for unexpected in unexpected_metas:
self.assertTrue(unexpected.document_id not in found_change_ids)
def test_checkpoint_with_multiple_topics(self):
feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE],
group_id='test-kafka-feed')
pillow_name = 'test-multi-topic-checkpoints'
checkpoint = PillowCheckpoint(pillow_name)
processor = CountingProcessor()
pillow = ConstructedPillow(
name=pillow_name,
checkpoint=checkpoint,
change_feed=feed,
processor=processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=1,
change_feed=feed))
offsets = feed.get_current_offsets()
self.assertEqual(set([topics.FORM, topics.CASE]), set(offsets.keys()))
# send a few changes to kafka so they should be picked up by the pillow
publish_stub_change(topics.FORM)
publish_stub_change(topics.FORM)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE_SQL)
pillow.process_changes(since=offsets, forever=False)
self.assertEqual(4, processor.count)
self.assertEqual(feed.get_current_checkpoint_offsets(),
pillow.get_last_checkpoint_sequence())
publish_stub_change(topics.FORM)
publish_stub_change(topics.FORM)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE)
publish_stub_change(topics.CASE_SQL)
pillow.process_changes(pillow.get_last_checkpoint_sequence(),
forever=False)
self.assertEqual(8, processor.count)
self.assertEqual(feed.get_current_checkpoint_offsets(),
pillow.get_last_checkpoint_sequence())
