def get_report_xform_to_elasticsearch_pillow(
pillow_id='ReportXFormToElasticsearchPillow',
num_processes=1,
process_num=0,
**kwargs):
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'ReportXFormToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, REPORT_XFORM_INDEX_INFO, topics.FORM_TOPICS)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=REPORT_XFORM_INDEX_INFO,
doc_prep_fn=transform_xform_for_report_forms_index,
doc_filter_fn=report_xform_filter)
kafka_change_feed = KafkaChangeFeed(topics=topics.FORM_TOPICS,
client_id='report-forms-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=kafka_change_feed),
)
def get_report_case_to_elasticsearch_pillow(
pillow_id='ReportCaseToElasticsearchPillow',
num_processes=1,
process_num=0,
**kwargs):
assert pillow_id == 'ReportCaseToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, REPORT_CASE_INDEX_INFO, topics.CASE_TOPICS)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=REPORT_CASE_INDEX_INFO,
doc_prep_fn=transform_case_to_report_es,
doc_filter_fn=report_case_filter,
)
kafka_change_feed = KafkaChangeFeed(topics=topics.CASE_TOPICS,
group_id='report-cases-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=kafka_change_feed),
)
def get_group_pillow(pillow_id='GroupPillow',
num_processes=1,
process_num=0,
**kwargs):
"""
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
assert pillow_id == 'GroupPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, GROUP_INDEX_INFO, [topics.GROUP])
processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=GROUP_INDEX_INFO,
)
change_feed = KafkaChangeFeed(topics=[topics.GROUP],
group_id='groups-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow',
num_processes=1,
process_num=0,
**kwargs):
"""Domain pillow to replicate documents to ES
Processors:
- :py:class:`pillowtop.processors.elastic.ElasticProcessor`
"""
assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, DOMAIN_INDEX_INFO, [topics.DOMAIN])
domain_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
doc_prep_fn=transform_domain_for_elasticsearch,
)
change_feed = KafkaChangeFeed(topics=[topics.DOMAIN],
client_id='domains-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=domain_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_user_pillow(pillow_id='UserPillow',
num_processes=1,
process_num=0,
**kwargs):
assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, USER_INDEX_INFO, topics.USER_TOPICS)
user_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_prep_fn=transform_user_for_elasticsearch,
)
change_feed = KafkaChangeFeed(topics=topics.USER_TOPICS,
group_id='users-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=user_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_user_pillow_old(pillow_id='UserPillow',
num_processes=1,
process_num=0,
**kwargs):
"""Processes users and sends them to ES.
Processors:
- :py:func:`pillowtop.processors.elastic.ElasticProcessor`
"""
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, USER_INDEX_INFO, topics.USER_TOPICS)
user_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_prep_fn=transform_user_for_elasticsearch,
)
change_feed = KafkaChangeFeed(topics=topics.USER_TOPICS,
client_id='users-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=user_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_case_to_report_es_processor():
return ElasticProcessor(
elasticsearch=get_es_new(),
index_info=REPORT_CASE_INDEX_INFO,
doc_prep_fn=transform_case_to_report_es,
doc_filter_fn=report_case_filter,
)
def get_app_to_elasticsearch_pillow(
pillow_id='ApplicationToElasticsearchPillow',
num_processes=1,
process_num=0,
**kwargs):
"""App pillow
Processors:
- :py:class:`pillowtop.processors.elastic.BulkElasticProcessor`
"""
assert pillow_id == 'ApplicationToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, APP_INDEX_INFO, [topics.APP])
app_processor = ElasticProcessor(elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_prep_fn=transform_app_for_es)
change_feed = KafkaChangeFeed(topics=[topics.APP],
client_id='apps-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=app_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_group_to_elasticsearch_processor():
"""
This processor adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
return ElasticProcessor(
elasticsearch=get_es_new(),
index_info=GROUP_INDEX_INFO,
)
def get_group_to_elasticsearch_processor():
"""Inserts group changes into ES
Reads from:
- Kafka topics: group
- Group data source (CouchDB)
Writes to:
- GroupES index
"""
return ElasticProcessor(
elasticsearch=get_es_new(),
index_info=GROUP_INDEX_INFO,
)
def get_app_to_elasticsearch_pillow(pillow_id='ApplicationToElasticsearchPillow'):
assert pillow_id == 'ApplicationToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, APP_INDEX_INFO)
app_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_prep_fn=transform_app_for_es
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[topics.APP], group_id='apps-to-es'),
processor=app_processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow'):
assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, DOMAIN_INDEX_INFO)
domain_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
doc_prep_fn=transform_domain_for_elasticsearch)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[DOMAIN], group_id='domains-to-es'),
processor=domain_processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
),
)
def get_user_pillow(pillow_id='UserPillow'):
assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, USER_INDEX_INFO)
user_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_prep_fn=transform_user_for_elasticsearch,
)
change_feed = KafkaChangeFeed(topics=[COMMCARE_USER, WEB_USER],
group_id='users-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=user_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_report_xform_to_elasticsearch_pillow(
pillow_id='ReportXFormToElasticsearchPillow', **kwargs):
assert pillow_id == 'ReportXFormToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, REPORT_XFORM_INDEX_INFO)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=REPORT_XFORM_INDEX_INFO,
doc_prep_fn=transform_xform_for_report_forms_index,
doc_filter_fn=report_xform_filter)
kafka_change_feed = KafkaChangeFeed(topics=topics.FORM_TOPICS,
group_id='report-forms-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=kafka_change_feed),
)
def get_user_es_processor():
return ElasticProcessor(
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_prep_fn=transform_user_for_elasticsearch,
)
