def get_group_to_user_pillow(pillow_id='GroupToUserPillow'):
assert pillow_id == 'GroupToUserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO)
processor = GroupsToUsersProcessor()
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[GROUP], group_id='groups-to-users'),
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_unknown_users_pillow(pillow_id="unknown-users-pillow"):
"""
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO)
processor = UnknownUsersProcessor()
change_feed = KafkaChangeFeed(topics=[FORM, FORM_SQL], group_id="unknown-users")
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_user_pillow(pillow_id="UserPillow"):
assert pillow_id == "UserPillow", "Pillow ID is not allowed to change"
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO)
user_processor = ElasticProcessor(
elasticsearch=get_es_new(), index_info=USER_INDEX_INFO, doc_prep_fn=transform_user_for_elasticsearch
)
change_feed = KafkaChangeFeed(topics=[COMMCARE_USER, WEB_USER], group_id="users-to-es")
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=user_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_group_to_user_pillow(pillow_id='GroupToUserPillow', num_processes=1, process_num=0, **kwargs):
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'GroupToUserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, [topics.GROUP])
processor = GroupsToUsersProcessor()
change_feed = KafkaChangeFeed(
topics=[topics.GROUP], client_id='groups-to-users', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=10, change_feed=change_feed
),
)
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow'):
assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, DOMAIN_INDEX_INFO)
domain_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
doc_prep_fn=transform_domain_for_elasticsearch
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[DOMAIN], group_id='domains-to-es'),
processor=domain_processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_ledger_to_elasticsearch_pillow(pillow_id='LedgerToElasticsearchPillow'):
assert pillow_id == 'LedgerToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, LEDGER_INDEX_INFO)
processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=LEDGER_INDEX_INFO,
doc_prep_fn=_prepare_ledger_for_es
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='ledgers-to-es'),
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100
),
)
def get_sql_sms_pillow(pillow_id='SqlSMSPillow'):
assert pillow_id == 'SqlSMSPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, SMS_INDEX_INFO)
processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=SMS_INDEX_INFO,
doc_prep_fn=lambda x: x
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[topics.SMS], group_id=SMS_PILLOW_KAFKA_CONSUMER_GROUP_ID),
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_app_to_elasticsearch_pillow(pillow_id='ApplicationToElasticsearchPillow'):
assert pillow_id == 'ApplicationToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, APP_INDEX_INFO)
app_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_prep_fn=transform_app_for_es
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[topics.APP], group_id='apps-to-es'),
processor=app_processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_case_to_elasticsearch_pillow(pillow_id='CaseToElasticsearchPillow'):
assert pillow_id == 'CaseToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, CASE_INDEX_INFO)
case_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_prep_fn=transform_case_for_elasticsearch
)
kafka_change_feed = KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='cases-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=case_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed
),
)
def get_report_xform_to_elasticsearch_pillow(pillow_id='ReportXFormToElasticsearchPillow'):
assert pillow_id == 'ReportXFormToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, REPORT_XFORM_INDEX_INFO)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=REPORT_XFORM_INDEX_INFO,
doc_prep_fn=transform_xform_for_report_forms_index,
doc_filter_fn=report_xform_filter
)
kafka_change_feed = KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='report-forms-to-es')
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=MultiTopicCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed
),
)
def get_group_pillow(pillow_id='GroupPillow'):
"""
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
assert pillow_id == 'GroupPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, GROUP_INDEX_INFO)
processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=GROUP_INDEX_INFO,
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=KafkaChangeFeed(topics=[GROUP], group_id='groups-to-es'),
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_unknown_users_pillow(pillow_id='unknown-users-pillow', num_processes=1, process_num=0, **kwargs):
"""
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, topics.FORM_TOPICS)
processor = UnknownUsersProcessor()
change_feed = KafkaChangeFeed(
topics=topics.FORM_TOPICS, client_id='unknown-users', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_group_pillow_old(pillow_id='GroupPillow', num_processes=1, process_num=0, **kwargs):
"""
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
assert pillow_id == 'GroupPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, GROUP_INDEX_INFO, [topics.GROUP])
processor = get_group_to_elasticsearch_processor()
change_feed = KafkaChangeFeed(
topics=[topics.GROUP], client_id='groups-to-es', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=10, change_feed=change_feed
),
)
def get_domain_kafka_to_elasticsearch_pillow(pillow_id='KafkaDomainPillow', num_processes=1,
process_num=0, **kwargs):
assert pillow_id == 'KafkaDomainPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, DOMAIN_INDEX_INFO, [topics.DOMAIN])
domain_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
doc_prep_fn=transform_domain_for_elasticsearch
)
change_feed = KafkaChangeFeed(
topics=[topics.DOMAIN], client_id='domains-to-es', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=domain_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_user_pillow_old(pillow_id='UserPillow', num_processes=1, process_num=0, **kwargs):
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'UserPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, topics.USER_TOPICS)
user_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_prep_fn=transform_user_for_elasticsearch,
)
change_feed = KafkaChangeFeed(
topics=topics.USER_TOPICS, client_id='users-to-es', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=user_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_user_pillow(pillow_id='user-pillow', num_processes=1, process_num=0,
skip_ucr=False, **kwargs):
# Pillow that sends users to ES and UCR
assert pillow_id == 'user-pillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, USER_INDEX_INFO, topics.USER_TOPICS)
user_processor = get_user_es_processor()
ucr_processor = ConfigurableReportPillowProcessor(
data_source_providers=[DynamicDataSourceProvider('CommCareUser'), StaticDataSourceProvider('CommCareUser')],
)
change_feed = KafkaChangeFeed(
topics=topics.USER_TOPICS, client_id='users-to-es', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=[user_processor] if skip_ucr else [ucr_processor, user_processor],
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_sql_sms_pillow(pillow_id='SqlSMSPillow', num_processes=1, process_num=0, **kwargs):
assert pillow_id == 'SqlSMSPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, SMS_INDEX_INFO, [topics.SMS])
processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=SMS_INDEX_INFO,
doc_prep_fn=lambda x: x
)
change_feed = KafkaChangeFeed(
topics=[topics.SMS], client_id='sql-sms-to-es',
num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed
),
)
def get_case_search_to_elasticsearch_pillow(pillow_id='CaseSearchToElasticsearchPillow', num_processes=1,
process_num=0, **kwargs):
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'CaseSearchToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, CASE_SEARCH_INDEX_INFO, topics.CASE_TOPICS)
case_processor = CaseSearchPillowProcessor(
elasticsearch=get_es_new(),
index_info=CASE_SEARCH_INDEX_INFO,
doc_prep_fn=transform_case_for_elasticsearch
)
change_feed = KafkaChangeFeed(
topics=topics.CASE_TOPICS, client_id='cases-to-es', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=case_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=change_feed,
),
)
def get_xform_to_elasticsearch_pillow(pillow_id='XFormToElasticsearchPillow', num_processes=1,
process_num=0, **kwargs):
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'XFormToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, XFORM_INDEX_INFO, FORM_TOPICS)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_prep_fn=transform_xform_for_elasticsearch,
doc_filter_fn=xform_pillow_filter,
)
kafka_change_feed = KafkaChangeFeed(
topics=FORM_TOPICS, client_id='forms-to-es', num_processes=num_processes, process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed
),
)
def get_report_case_to_elasticsearch_pillow(pillow_id='ReportCaseToElasticsearchPillow',
num_processes=1, process_num=0, **kwargs):
# todo; To remove after full rollout of https://github.com/dimagi/commcare-hq/pull/21329/
assert pillow_id == 'ReportCaseToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(pillow_id, REPORT_CASE_INDEX_INFO, topics.CASE_TOPICS)
form_processor = ElasticProcessor(
elasticsearch=get_es_new(),
index_info=REPORT_CASE_INDEX_INFO,
doc_prep_fn=transform_case_to_report_es,
doc_filter_fn=report_case_filter,
)
kafka_change_feed = KafkaChangeFeed(
topics=topics.CASE_TOPICS, client_id='report-cases-to-es', num_processes=num_processes,
process_num=process_num
)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=kafka_change_feed,
processor=form_processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100, change_feed=kafka_change_feed
),
)
def get_unknown_users_pillow(pillow_id='unknown-users-pillow',
num_processes=1,
process_num=0,
**kwargs):
"""
This pillow adds users from xform submissions that come in to the User Index if they don't exist in HQ
"""
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, USER_INDEX_INFO, topics.FORM_TOPICS)
processor = UnknownUsersProcessor()
change_feed = KafkaChangeFeed(topics=topics.FORM_TOPICS,
group_id='unknown-users',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_sql_sms_pillow(pillow_id='SqlSMSPillow',
num_processes=1,
process_num=0,
**kwargs):
assert pillow_id == 'SqlSMSPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, SMS_INDEX_INFO, [topics.SMS])
processor = ElasticProcessor(elasticsearch=get_es_new(),
index_info=SMS_INDEX_INFO,
doc_prep_fn=lambda x: x)
change_feed = KafkaChangeFeed(topics=[topics.SMS],
client_id='sql-sms-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
def get_ledger_to_elasticsearch_pillow(pillow_id='LedgerToElasticsearchPillow',
num_processes=1,
process_num=0,
**kwargs):
assert pillow_id == 'LedgerToElasticsearchPillow', 'Pillow ID is not allowed to change'
checkpoint = get_checkpoint_for_elasticsearch_pillow(
pillow_id, LEDGER_INDEX_INFO, [topics.LEDGER])
processor = ElasticProcessor(elasticsearch=get_es_new(),
index_info=LEDGER_INDEX_INFO,
doc_prep_fn=_prepare_ledger_for_es)
change_feed = KafkaChangeFeed(topics=[topics.LEDGER],
group_id='ledgers-to-es',
num_processes=num_processes,
process_num=process_num)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=KafkaCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
change_feed=change_feed),
)
