def test_checkpoint_with_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') pillow_name = 'test-multi-topic-checkpoints' checkpoint = PillowCheckpoint(pillow_name) processor = CountingProcessor() pillow = ConstructedPillow( name=pillow_name, document_store=None, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed ) ) offsets = feed.get_current_offsets() self.assertEqual(set([topics.FORM, topics.CASE]), set(offsets.keys())) # send a few changes to kafka so they should be picked up by the pillow publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(since=offsets, forever=False) self.assertEqual(4, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence()) publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(8, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
def test_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id="test-kafka-feed") self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False)))) producer = KeyedProducer(get_kafka_client_or_none()) offsets = feed.get_current_offsets() send_to_kafka( producer, topics.FORM, ChangeMeta(document_id="1", data_source_type="form", data_source_name="form") ) send_to_kafka( producer, topics.CASE, ChangeMeta(document_id="2", data_source_type="case", data_source_name="case") ) send_to_kafka( producer, topics.FORM_SQL, ChangeMeta(document_id="3", data_source_type="form-sql", data_source_name="form-sql"), ) send_to_kafka( producer, topics.CASE_SQL, ChangeMeta(document_id="4", data_source_type="case-sql", data_source_name="case-sql"), ) changes = list(feed.iter_changes(since=offsets, forever=False)) self.assertEqual(2, len(changes)) self.assertEqual(set(["1", "2"]), set([change.id for change in changes]))
def test_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False)))) offsets = feed.get_current_offsets() expected_metas = [publish_stub_change(topics.FORM), publish_stub_change(topics.CASE)] unexpected_metas = [publish_stub_change(topics.FORM_SQL), publish_stub_change(topics.CASE_SQL)] changes = list(feed.iter_changes(since=offsets, forever=False)) self.assertEqual(2, len(changes)) found_change_ids = set([change.id for change in changes]) self.assertEqual(set([meta.document_id for meta in expected_metas]), found_change_ids) for unexpected in unexpected_metas: self.assertTrue(unexpected.document_id not in found_change_ids)
def test_multiple_topics_with_partial_checkpoint(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False)))) offsets = {'form': feed.get_current_offsets()['form']} expected_metas = [publish_stub_change(topics.FORM), publish_stub_change(topics.CASE)] changes = list(feed.iter_changes(since=offsets, forever=False)) # should include at least the form and the case (may have more than one case since not # specifying a checkpoint rewinds it to the beginning of the feed) self.assertTrue(len(changes) > 1) found_change_ids = set([change.id for change in changes]) for expected_id in set([meta.document_id for meta in expected_metas]): self.assertTrue(expected_id in found_change_ids)
def test_multiple_topics_with_partial_checkpoint(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') self.assertEqual( 0, len(list(feed.iter_changes(since=None, forever=False)))) offsets = {'form': feed.get_current_offsets()['form']} expected_metas = [ publish_stub_change(topics.FORM), publish_stub_change(topics.CASE) ] changes = list(feed.iter_changes(since=offsets, forever=False)) # should include at least the form and the case (may have more than one case since not # specifying a checkpoint rewinds it to the beginning of the feed) self.assertTrue(len(changes) > 1) found_change_ids = set([change.id for change in changes]) for expected_id in set([meta.document_id for meta in expected_metas]): self.assertTrue(expected_id in found_change_ids)
def test_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') self.assertEqual( 0, len(list(feed.iter_changes(since=None, forever=False)))) offsets = feed.get_current_offsets() expected_metas = [ publish_stub_change(topics.FORM), publish_stub_change(topics.CASE) ] unexpected_metas = [ publish_stub_change(topics.FORM_SQL), publish_stub_change(topics.CASE_SQL) ] changes = list(feed.iter_changes(since=offsets, forever=False)) self.assertEqual(2, len(changes)) found_change_ids = set([change.id for change in changes]) self.assertEqual(set([meta.document_id for meta in expected_metas]), found_change_ids) for unexpected in unexpected_metas: self.assertTrue(unexpected.document_id not in found_change_ids)
def test_checkpoint_with_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') pillow_name = 'test-multi-topic-checkpoints' checkpoint = PillowCheckpoint(pillow_name) processor = CountingProcessor() pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed)) offsets = feed.get_current_offsets() self.assertEqual(set([topics.FORM, topics.CASE]), set(offsets.keys())) # send a few changes to kafka so they should be picked up by the pillow publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(since=offsets, forever=False) self.assertEqual(4, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence()) publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(8, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())