def test_checkpoint_with_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') pillow_name = 'test-multi-topic-checkpoints' checkpoint = PillowCheckpoint(pillow_name, feed.sequence_format) processor = CountingProcessor() pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed ) ) offsets = feed.get_latest_offsets() self.assertEqual(set([(topics.FORM, 0), (topics.CASE, 0)]), set(offsets.keys())) # send a few changes to kafka so they should be picked up by the pillow publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(since=offsets, forever=False) self.assertEqual(4, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence()) publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(8, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
def test_checkpoint_with_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') pillow_name = 'test-multi-topic-checkpoints' checkpoint = PillowCheckpoint(pillow_name) processor = CountingProcessor() pillow = ConstructedPillow( name=pillow_name, document_store=None, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed ) ) offsets = feed.get_current_offsets() self.assertEqual(set([topics.FORM, topics.CASE]), set(offsets.keys())) # send a few changes to kafka so they should be picked up by the pillow publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(since=offsets, forever=False) self.assertEqual(4, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence()) publish_stub_change(topics.FORM) publish_stub_change(topics.FORM) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE) publish_stub_change(topics.CASE_SQL) pillow.process_changes(pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(8, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), pillow.get_last_checkpoint_sequence())
def test_dont_create_checkpoint_past_current(self): pillow_name = 'test-checkpoint-reset' # initialize change feed and pillow feed = KafkaChangeFeed(topics=topics.USER_TOPICS, group_id='test-kafka-feed') checkpoint = PillowCheckpoint(pillow_name, feed.sequence_format) processor = CountingProcessor() pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed ) ) original_kafka_offsets = feed.get_latest_offsets() current_kafka_offsets = deepcopy(original_kafka_offsets) self.assertEqual(feed.get_current_checkpoint_offsets(), {}) self.assertEqual(pillow.get_last_checkpoint_sequence(), {}) publish_stub_change(topics.COMMCARE_USER) # the following line causes tests to fail if you have multiple partitions current_kafka_offsets[(topics.COMMCARE_USER, 0)] += 1 pillow.process_changes(since=original_kafka_offsets, forever=False) self.assertEqual(1, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), current_kafka_offsets)
def test_dont_create_checkpoint_past_current(self): pillow_name = 'test-checkpoint-reset' # initialize change feed and pillow feed = KafkaChangeFeed(topics=topics.USER_TOPICS, client_id='test-kafka-feed') checkpoint = PillowCheckpoint(pillow_name, feed.sequence_format) processor = CountingProcessor() pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed ) ) original_kafka_offsets = feed.get_latest_offsets() current_kafka_offsets = deepcopy(original_kafka_offsets) self.assertEqual(feed.get_current_checkpoint_offsets(), {}) self.assertEqual(pillow.get_last_checkpoint_sequence(), {}) publish_stub_change(topics.COMMCARE_USER) # the following line causes tests to fail if you have multiple partitions current_kafka_offsets[(topics.COMMCARE_USER, 0)] += 1 pillow.process_changes(since=original_kafka_offsets, forever=False) self.assertEqual(1, processor.count) self.assertEqual(feed.get_current_checkpoint_offsets(), current_kafka_offsets)
def test_basic(self): # setup feed = KafkaChangeFeed(topics=[topics.CASE], client_id='test-kafka-feed') pillow_name = 'test-chunked-processing' checkpoint = PillowCheckpoint(pillow_name, feed.sequence_format) processor = ChunkedCountProcessor() original_process_change = processor.process_change original_process_changes_chunk = processor.process_changes_chunk pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed), processor_chunk_size=2) since = feed.get_latest_offsets() self._produce_changes(2) # pillow should use process_changes_chunk (make process_change raise an exception for test) processor.process_change = MagicMock(side_effect=Exception('_')) pillow.process_changes(since=since, forever=False) self.assertEqual(processor.count, 2) self._produce_changes(2) # if process_changes_chunk raises exception, pillow should use process_change processor.process_change = original_process_change processor.process_changes_chunk = MagicMock(side_effect=Exception('_')) pillow.process_changes(since=pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(processor.count, 4) self._produce_changes(1) # offsets after full chunk should still be processed processor.process_change = MagicMock(side_effect=Exception('_')) processor.process_changes_chunk = original_process_changes_chunk pillow.process_changes(since=pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(processor.count, 5)
def test_basic(self): # setup feed = KafkaChangeFeed(topics=[topics.CASE], client_id='test-kafka-feed') pillow_name = 'test-chunked-processing' checkpoint = PillowCheckpoint(pillow_name, feed.sequence_format) processor = ChunkedCountProcessor() original_process_change = processor.process_change original_process_changes_chunk = processor.process_changes_chunk pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed ), processor_chunk_size=2 ) since = feed.get_latest_offsets() self._produce_changes(2) # pillow should use process_changes_chunk (make process_change raise an exception for test) processor.process_change = MagicMock(side_effect=Exception('_')) pillow.process_changes(since=since, forever=False) self.assertEqual(processor.count, 2) self._produce_changes(2) # if process_changes_chunk raises exception, pillow should use process_change processor.process_change = original_process_change processor.process_changes_chunk = MagicMock(side_effect=Exception('_')) pillow.process_changes(since=pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(processor.count, 4) self._produce_changes(1) # offsets after full chunk should still be processed processor.process_change = MagicMock(side_effect=Exception('_')) processor.process_changes_chunk = original_process_changes_chunk pillow.process_changes(since=pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(processor.count, 5)