def test_expired_checkpoint_iteration_strict(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], client_id='test-kafka-feed', strict=True) first_available_offsets = get_multi_topic_first_available_offsets([topics.FORM, topics.CASE]) since = { topic_partition: offset - 1 for topic_partition, offset in first_available_offsets.items() } with self.assertRaises(UnavailableKafkaOffset): next(feed.iter_changes(since=since, forever=False))
def setUp(self): self.processor = CountingProcessor() self.pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'), processor=self.processor) self.original_process_change = self.pillow.process_change
def get_case_pillow( pillow_id='case-pillow', ucr_division=None, include_ucrs=None, exclude_ucrs=None, num_processes=1, process_num=0, ucr_configs=None, skip_ucr=False, processor_chunk_size=DEFAULT_PROCESSOR_CHUNK_SIZE, topics=None, **kwargs): """ Return a pillow that processes cases. The processors include, UCR and elastic processors Args: skip_ucr: Can be set to True to avoid passing UCR processor, useful for tests """ if topics: assert set(topics).issubset(CASE_TOPICS), "This is a pillow to process cases only" topics = topics or CASE_TOPICS change_feed = KafkaChangeFeed( topics, client_id=pillow_id, num_processes=num_processes, process_num=process_num ) ucr_processor = ConfigurableReportPillowProcessor( data_source_providers=[DynamicDataSourceProvider('CommCareCase'), StaticDataSourceProvider('CommCareCase')], ucr_division=ucr_division, include_ucrs=include_ucrs, exclude_ucrs=exclude_ucrs, run_migrations=(process_num == 0), # only first process runs migrations ) if ucr_configs: ucr_processor.bootstrap(ucr_configs) case_to_es_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_prep_fn=transform_case_for_elasticsearch ) case_search_processor = get_case_search_processor() checkpoint_id = "{}-{}-{}-{}".format( pillow_id, CASE_INDEX_INFO.index, case_search_processor.index_info.index, 'messaging-sync') checkpoint = KafkaPillowCheckpoint(checkpoint_id, topics) event_handler = KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1000, change_feed=change_feed, checkpoint_callback=ucr_processor ) processors = [case_to_es_processor, CaseMessagingSyncProcessor()] if settings.RUN_CASE_SEARCH_PILLOW: processors.append(case_search_processor) if not settings.ENTERPRISE_MODE: processors.append(get_case_to_report_es_processor()) if not skip_ucr: # this option is useful in tests to avoid extra UCR setup where unneccessary processors = [ucr_processor] + processors return ConstructedPillow( name=pillow_id, change_feed=change_feed, checkpoint=checkpoint, change_processed_event_handler=event_handler, processor=processors, processor_chunk_size=processor_chunk_size )
def test_non_expired_checkpoint_iteration_strict(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed', strict=True) first_avaliable_offsets = get_multi_topic_first_available_offsets( [topics.FORM, topics.CASE]) since = { topic: first_available for topic, first_available in first_avaliable_offsets.items() } feed.iter_changes(since=since, forever=False).next()
def get_main_blob_deletion_pillow(pillow_id): """Get blob deletion pillow for the main couch database Using the KafkaChangeFeed ties this to the main couch database. """ return _get_blob_deletion_pillow( pillow_id, get_db(None), PillowCheckpoint('kafka-blob-deletion-pillow-checkpoint'), KafkaChangeFeed(topics=[topics.META], group_id='blob-deletion-group'), )
def test_multiple_topics(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False)))) offsets = feed.get_latest_offsets() expected_metas = [publish_stub_change(topics.FORM), publish_stub_change(topics.CASE)] unexpected_metas = [publish_stub_change(topics.FORM_SQL), publish_stub_change(topics.CASE_SQL)] changes = list(feed.iter_changes(since=offsets, forever=False)) self.assertEqual(2, len(changes)) found_change_ids = set([change.id for change in changes]) self.assertEqual(set([meta.document_id for meta in expected_metas]), found_change_ids) for unexpected in unexpected_metas: self.assertTrue(unexpected.document_id not in found_change_ids)
def test_multiple_topics_with_partial_checkpoint(self): feed = KafkaChangeFeed(topics=[topics.FORM, topics.CASE], group_id='test-kafka-feed') self.assertEqual(0, len(list(feed.iter_changes(since=None, forever=False)))) offsets = {'form': feed.get_latest_offsets()['form']} expected_metas = [publish_stub_change(topics.FORM), publish_stub_change(topics.CASE)] changes = list(feed.iter_changes(since=offsets, forever=False)) # should include at least the form and the case (may have more than one case since not # specifying a checkpoint rewinds it to the beginning of the feed) self.assertTrue(len(changes) > 1) found_change_ids = set([change.id for change in changes]) for expected_id in set([meta.document_id for meta in expected_metas]): self.assertTrue(expected_id in found_change_ids)
def setUpClass(cls): super(KafkaPublishingTest, cls).setUpClass() cls.processor = TestProcessor() cls.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM_SQL], client_id='test-kafka-form-feed'), processor=cls.processor) cls.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE_SQL], client_id='test-kafka-case-feed'), processor=cls.processor) cls.process_form_changes = process_pillow_changes( 'DefaultChangeFeedPillow') cls.process_form_changes.add_pillow(cls.form_pillow) cls.process_case_changes = process_pillow_changes( 'DefaultChangeFeedPillow') cls.process_case_changes.add_pillow(cls.case_pillow)
def setUp(self): super(KafkaPublishingTest, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.form_accessors = FormAccessors(domain=self.domain) self.processor = TestProcessor() self.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'), processor=self.processor) self.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'), processor=self.processor) self.ledger_pillow = ConstructedPillow( name='test-kafka-ledger-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'), processor=self.processor)
def get_group_to_user_pillow(pillow_id='GroupToUserPillow', num_processes=1, process_num=0, **kwargs): assert pillow_id == 'GroupToUserPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, [topics.GROUP]) processor = GroupsToUsersProcessor() change_feed = KafkaChangeFeed( topics=[topics.GROUP], group_id='groups-to-users', num_processes=num_processes, process_num=process_num ) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed ), )
def get_group_to_user_pillow(pillow_id='GroupToUserPillow'): assert pillow_id == 'GroupToUserPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, USER_INDEX_INFO) processor = GroupsToUsersProcessor() return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[GROUP], group_id='groups-to-users'), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def __init__(self, processor, pillow_name): change_feed = KafkaChangeFeed(topics.ALL, group_id=pillow_name) checkpoint = PillowCheckpoint(pillow_name) event_handler = MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1000, change_feed=change_feed ) super(ConfigurableReportKafkaPillow, self).__init__( name=pillow_name, change_feed=change_feed, processor=processor, checkpoint=checkpoint, change_processed_event_handler=event_handler ) # set by the superclass constructor assert self._processor is not None assert self._processor.bootstrapped is not None
def get_group_to_user_pillow(pillow_id='GroupToUserPillow', num_processes=1, process_num=0, **kwargs): # todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/ assert pillow_id == 'GroupToUserPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, [topics.GROUP]) processor = GroupsToUsersProcessor() change_feed = KafkaChangeFeed( topics=[topics.GROUP], client_id='groups-to-users', num_processes=num_processes, process_num=process_num ) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=10, change_feed=change_feed ), )
def get_user_pillow(pillow_id='user-pillow', num_processes=1, dedicated_migration_process=False, process_num=0, skip_ucr=False, processor_chunk_size=DEFAULT_PROCESSOR_CHUNK_SIZE, **kwargs): """Processes users and sends them to ES and UCRs. Processors: - :py:func:`pillowtop.processors.elastic.BulkElasticProcessor` - :py:func:`corehq.apps.userreports.pillow.ConfigurableReportPillowProcessor` """ # Pillow that sends users to ES and UCR assert pillow_id == 'user-pillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, USER_INDEX_INFO, topics.USER_TOPICS) user_processor = get_user_es_processor() ucr_processor = get_ucr_processor( data_source_providers=[ DynamicDataSourceProvider('CommCareUser'), StaticDataSourceProvider('CommCareUser') ], run_migrations=( process_num == 0), # only first process runs migrations, ) change_feed = KafkaChangeFeed( topics=topics.USER_TOPICS, client_id='users-to-es', num_processes=num_processes, process_num=process_num, dedicated_migration_process=dedicated_migration_process) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=[user_processor] if skip_ucr else [ucr_processor, user_processor], change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), processor_chunk_size=processor_chunk_size, process_num=process_num, is_dedicated_migration_process=dedicated_migration_process and (process_num == 0))
def get_ledger_to_elasticsearch_pillow( pillow_id='LedgerToElasticsearchPillow'): assert pillow_id == 'LedgerToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, LEDGER_INDEX_INFO) processor = ElasticProcessor(elasticsearch=get_es_new(), index_info=LEDGER_INDEX_INFO, doc_prep_fn=_prepare_ledger_for_es) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='ledgers-to-es'), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100), )
def get_app_to_elasticsearch_pillow(pillow_id='ApplicationToElasticsearchPillow'): assert pillow_id == 'ApplicationToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, APP_INDEX_INFO) app_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=APP_INDEX_INFO, doc_prep_fn=transform_app_for_es ) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[topics.APP], group_id='apps-to-es'), processor=app_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_sql_sms_pillow(pillow_id='SqlSMSPillow'): assert pillow_id == 'SqlSMSPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, SMS_INDEX_INFO) processor = ElasticProcessor(elasticsearch=get_es_new(), index_info=SMS_INDEX_INFO, doc_prep_fn=lambda x: x) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed( topics=[topics.SMS], group_id=SMS_PILLOW_KAFKA_CONSUMER_GROUP_ID), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def __init__(self, indicator_class, processor): self.indicator_class = indicator_class self.kafka_topic = indicator_class().kafka_topic self.domains = processor.domains self.doc_type = processor.doc_type name = '{}Pillow'.format(indicator_class.__name__) checkpoint = PillowCheckpoint('fluff.{}.{}'.format(name, get_machine_id())) super(FluffPillow, self).__init__( name=name, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[self.kafka_topic], group_id=indicator_class.__name__), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1000, ) )
def get_case_to_elasticsearch_pillow(pillow_id='CaseToElasticsearchPillow'): assert pillow_id == 'CaseToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, CASE_INDEX_INFO) case_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_prep_fn=transform_case_for_elasticsearch ) kafka_change_feed = KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='cases-to-es') return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=kafka_change_feed, processor=case_processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed ), )
def handle(self, **options): since = options['from'] sleep = float(options['sleep'] or '.01') last_domain = None change_feed = KafkaChangeFeed(topics=[topics.FORM], group_id='form-feed') for change in change_feed.iter_changes(since=since, forever=True): if not change.deleted: # this is just helpful for demos to find domain transitions if change.metadata.domain != last_domain: last_domain = change.metadata.domain print(change.sequence_id, last_domain) metadata = change.metadata.to_json() if not options['compact']: metadata['country'] = _get_country(change.metadata.domain) message = RedisMessage(json.dumps(metadata)) RedisPublisher(facility='form-feed', broadcast=True).publish_message(message) time.sleep(sleep)
def get_user_sync_history_pillow( pillow_id='UpdateUserSyncHistoryPillow', num_processes=1, process_num=0, **kwargs): """ This gets a pillow which iterates through all synclogs """ change_feed = KafkaChangeFeed( topics=[topics.SYNCLOG_SQL], client_id=SYNCLOG_SQL_USER_SYNC_GROUP_ID, num_processes=num_processes, process_num=process_num) checkpoint = KafkaPillowCheckpoint(pillow_id, [topics.SYNCLOG_SQL]) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=UserSyncHistoryProcessor(), change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed ), )
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow'): assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, DOMAIN_INDEX_INFO) domain_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=DOMAIN_INDEX_INFO, doc_prep_fn=transform_domain_for_elasticsearch) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[DOMAIN], group_id='domains-to-es'), processor=domain_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_unknown_users_pillow(pillow_id='unknown-users-pillow', num_processes=1, process_num=0, **kwargs): """ This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ """ checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, topics.FORM_TOPICS) processor = UnknownUsersProcessor() change_feed = KafkaChangeFeed( topics=topics.FORM_TOPICS, group_id='unknown-users', num_processes=num_processes, process_num=process_num ) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed ), )
def get_unknown_users_pillow(pillow_id='unknown-users-pillow', num_processes=1, process_num=0, **kwargs): """ # todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/ This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ """ checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, topics.FORM_TOPICS) processor = UnknownUsersProcessor() change_feed = KafkaChangeFeed( topics=topics.FORM_TOPICS, client_id='unknown-users', num_processes=num_processes, process_num=process_num ) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed ), )
def __init__(self, indicator_name, kafka_topic, processor, domains=None, doc_type=None): self.kafka_topic = kafka_topic self.domains = domains or processor.domains self.doc_type = doc_type or processor.doc_type change_feed = KafkaChangeFeed(topics=[self.kafka_topic], client_id=indicator_name) name = '{}Pillow'.format(indicator_name) checkpoint = PillowCheckpoint('fluff.{}.{}'.format(name, get_machine_id()), change_feed.sequence_format) super(FluffPillow, self).__init__( name=name, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1000, change_feed=change_feed ) )
def get_report_case_to_elasticsearch_pillow(pillow_id='ReportCaseToElasticsearchPillow', **kwargs): assert pillow_id == 'ReportCaseToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, REPORT_CASE_INDEX_INFO) form_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=REPORT_CASE_INDEX_INFO, doc_prep_fn=transform_case_to_report_es, doc_filter_fn=report_case_filter, ) kafka_change_feed = KafkaChangeFeed(topics=topics.CASE_TOPICS, group_id='report-cases-to-es') return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=kafka_change_feed, processor=form_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed ), )
def get_unknown_users_pillow(pillow_id='unknown-users-pillow'): """ This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ """ checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, USER_INDEX_INFO) processor = UnknownUsersProcessor() change_feed = KafkaChangeFeed(topics=[FORM, FORM_SQL], group_id='unknown-users') return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_group_pillow(pillow_id='group-pillow', num_processes=1, process_num=0, **kwargs): assert pillow_id == 'group-pillow', 'Pillow ID is not allowed to change' to_user_es_processor = GroupsToUsersProcessor() to_group_es_processor = get_group_to_elasticsearch_processor() change_feed = KafkaChangeFeed( topics=[topics.GROUP], client_id='groups-to-users', num_processes=num_processes, process_num=process_num ) checkpoint_id = "{}-{}-{}".format( pillow_id, USER_INDEX, to_group_es_processor.index_info.index) checkpoint = KafkaPillowCheckpoint(checkpoint_id, [topics.GROUP]) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=[to_user_es_processor, to_group_es_processor], change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=10, change_feed=change_feed ), )
def test_basic(self): # setup feed = KafkaChangeFeed(topics=[topics.CASE], client_id='test-kafka-feed') pillow_name = 'test-chunked-processing' checkpoint = PillowCheckpoint(pillow_name, feed.sequence_format) processor = ChunkedCountProcessor() original_process_change = processor.process_change original_process_changes_chunk = processor.process_changes_chunk pillow = ConstructedPillow( name=pillow_name, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=1, change_feed=feed), processor_chunk_size=2) since = feed.get_latest_offsets() self._produce_changes(2) # pillow should use process_changes_chunk (make process_change raise an exception for test) processor.process_change = MagicMock(side_effect=Exception('_')) pillow.process_changes(since=since, forever=False) self.assertEqual(processor.count, 2) self._produce_changes(2) # if process_changes_chunk raises exception, pillow should use process_change processor.process_change = original_process_change processor.process_changes_chunk = MagicMock(side_effect=Exception('_')) pillow.process_changes(since=pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(processor.count, 4) self._produce_changes(1) # offsets after full chunk should still be processed processor.process_change = MagicMock(side_effect=Exception('_')) processor.process_changes_chunk = original_process_changes_chunk pillow.process_changes(since=pillow.get_last_checkpoint_sequence(), forever=False) self.assertEqual(processor.count, 5)
def get_xform_to_elasticsearch_pillow(pillow_id='XFormToElasticsearchPillow'): assert pillow_id == 'XFormToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, XFORM_INDEX_INFO) form_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=XFORM_INDEX_INFO, doc_prep_fn=transform_xform_for_elasticsearch, doc_filter_fn=xform_pillow_filter, ) kafka_change_feed = KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='forms-to-es') return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=kafka_change_feed, processor=form_processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed ), )