def get_report_xform_to_elasticsearch_pillow( pillow_id='ReportXFormToElasticsearchPillow', num_processes=1, process_num=0, **kwargs): # todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/ assert pillow_id == 'ReportXFormToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, REPORT_XFORM_INDEX_INFO, topics.FORM_TOPICS) form_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=REPORT_XFORM_INDEX_INFO, doc_prep_fn=transform_xform_for_report_forms_index, doc_filter_fn=report_xform_filter) kafka_change_feed = KafkaChangeFeed(topics=topics.FORM_TOPICS, client_id='report-forms-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=kafka_change_feed, processor=form_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed), )
def get_report_case_to_elasticsearch_pillow( pillow_id='ReportCaseToElasticsearchPillow', num_processes=1, process_num=0, **kwargs): assert pillow_id == 'ReportCaseToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, REPORT_CASE_INDEX_INFO, topics.CASE_TOPICS) form_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=REPORT_CASE_INDEX_INFO, doc_prep_fn=transform_case_to_report_es, doc_filter_fn=report_case_filter, ) kafka_change_feed = KafkaChangeFeed(topics=topics.CASE_TOPICS, group_id='report-cases-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=kafka_change_feed, processor=form_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed), )
def get_group_pillow(pillow_id='GroupPillow', num_processes=1, process_num=0, **kwargs): """ This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ """ assert pillow_id == 'GroupPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, GROUP_INDEX_INFO, [topics.GROUP]) processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=GROUP_INDEX_INFO, ) change_feed = KafkaChangeFeed(topics=[topics.GROUP], group_id='groups-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow', num_processes=1, process_num=0, **kwargs): """Domain pillow to replicate documents to ES Processors: - :py:class:`pillowtop.processors.elastic.ElasticProcessor` """ assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, DOMAIN_INDEX_INFO, [topics.DOMAIN]) domain_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=DOMAIN_INDEX_INFO, doc_prep_fn=transform_domain_for_elasticsearch, ) change_feed = KafkaChangeFeed(topics=[topics.DOMAIN], client_id='domains-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=domain_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_user_pillow(pillow_id='UserPillow', num_processes=1, process_num=0, **kwargs): assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, USER_INDEX_INFO, topics.USER_TOPICS) user_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=USER_INDEX_INFO, doc_prep_fn=transform_user_for_elasticsearch, ) change_feed = KafkaChangeFeed(topics=topics.USER_TOPICS, group_id='users-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=user_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_user_pillow_old(pillow_id='UserPillow', num_processes=1, process_num=0, **kwargs): """Processes users and sends them to ES. Processors: - :py:func:`pillowtop.processors.elastic.ElasticProcessor` """ # todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/ assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, USER_INDEX_INFO, topics.USER_TOPICS) user_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=USER_INDEX_INFO, doc_prep_fn=transform_user_for_elasticsearch, ) change_feed = KafkaChangeFeed(topics=topics.USER_TOPICS, client_id='users-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=user_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_case_to_report_es_processor(): return ElasticProcessor( elasticsearch=get_es_new(), index_info=REPORT_CASE_INDEX_INFO, doc_prep_fn=transform_case_to_report_es, doc_filter_fn=report_case_filter, )
def get_app_to_elasticsearch_pillow( pillow_id='ApplicationToElasticsearchPillow', num_processes=1, process_num=0, **kwargs): """App pillow Processors: - :py:class:`pillowtop.processors.elastic.BulkElasticProcessor` """ assert pillow_id == 'ApplicationToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, APP_INDEX_INFO, [topics.APP]) app_processor = ElasticProcessor(elasticsearch=get_es_new(), index_info=APP_INDEX_INFO, doc_prep_fn=transform_app_for_es) change_feed = KafkaChangeFeed(topics=[topics.APP], client_id='apps-to-es', num_processes=num_processes, process_num=process_num) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=app_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_group_to_elasticsearch_processor(): """ This processor adds users from xform submissions that come in to the User Index if they don't exist in HQ """ return ElasticProcessor( elasticsearch=get_es_new(), index_info=GROUP_INDEX_INFO, )
def get_group_to_elasticsearch_processor(): """Inserts group changes into ES Reads from: - Kafka topics: group - Group data source (CouchDB) Writes to: - GroupES index """ return ElasticProcessor( elasticsearch=get_es_new(), index_info=GROUP_INDEX_INFO, )
def get_app_to_elasticsearch_pillow(pillow_id='ApplicationToElasticsearchPillow'): assert pillow_id == 'ApplicationToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, APP_INDEX_INFO) app_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=APP_INDEX_INFO, doc_prep_fn=transform_app_for_es ) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[topics.APP], group_id='apps-to-es'), processor=app_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow'): assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, DOMAIN_INDEX_INFO) domain_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=DOMAIN_INDEX_INFO, doc_prep_fn=transform_domain_for_elasticsearch) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topics=[DOMAIN], group_id='domains-to-es'), processor=domain_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_user_pillow(pillow_id='UserPillow'): assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, USER_INDEX_INFO) user_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=USER_INDEX_INFO, doc_prep_fn=transform_user_for_elasticsearch, ) change_feed = KafkaChangeFeed(topics=[COMMCARE_USER, WEB_USER], group_id='users-to-es') return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=user_processor, change_processed_event_handler=MultiTopicCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed), )
def get_report_xform_to_elasticsearch_pillow( pillow_id='ReportXFormToElasticsearchPillow', **kwargs): assert pillow_id == 'ReportXFormToElasticsearchPillow', 'Pillow ID is not allowed to change' checkpoint = get_checkpoint_for_elasticsearch_pillow( pillow_id, REPORT_XFORM_INDEX_INFO) form_processor = ElasticProcessor( elasticsearch=get_es_new(), index_info=REPORT_XFORM_INDEX_INFO, doc_prep_fn=transform_xform_for_report_forms_index, doc_filter_fn=report_xform_filter) kafka_change_feed = KafkaChangeFeed(topics=topics.FORM_TOPICS, group_id='report-forms-to-es') return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=kafka_change_feed, processor=form_processor, change_processed_event_handler=KafkaCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed), )
def get_user_es_processor(): return ElasticProcessor( elasticsearch=get_es_new(), index_info=USER_INDEX_INFO, doc_prep_fn=transform_user_for_elasticsearch, )