def get_ledger_v2_reindexer():
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format(
LEDGER_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, LedgerReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=LEDGER_INDEX_INFO,
doc_transform=_prepare_ledger_for_es)
def get_sql_case_reindexer():
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format(
CASE_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, CaseReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch)
def build(self):
iteration_key = f"SmsToElasticsearchPillow_{SMS_INDEX_INFO.index}_reindexer"
reindex_accessor = SMSReindexAccessor()
doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor)
return ResumableBulkElasticPillowReindexer(doc_provider,
elasticsearch=get_es_new(),
index_info=SMS_INDEX_INFO,
pillow=get_sql_sms_pillow(),
**self.options)
def get_sql_form_reindexer():
iteration_key = "SqlXFormToElasticsearchPillow_{}_reindexer".format(
XFORM_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, FormReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_transform=transform_xform_for_elasticsearch)
def get_app_reindexer():
iteration_key = "ApplicationToElasticsearchPillow_{}_reindexer".format(APP_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(iteration_key, [Application, RemoteApp])
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_transform=transform_app_for_es,
pillow=get_app_to_elasticsearch_pillow(),
)
def get_couch_case_reindexer():
iteration_key = "CouchCaseToElasticsearchPillow_{}_reindexer".format(
CASE_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(
iteration_key,
doc_type_tuples=[CommCareCase, ("CommCareCase-Deleted", CommCareCase)])
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
pillow=get_case_to_elasticsearch_pillow())
def build(self):
domain = self.options.pop('domain', None)
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}".format(
LEDGER_INDEX_INFO.index, domain or 'all')
doc_provider = SqlDocumentProvider(
iteration_key, LedgerReindexAccessor(domain=domain))
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=LEDGER_INDEX_INFO,
doc_transform=_prepare_ledger_for_es,
**self.options)
def build(self):
iteration_key = "DomainToElasticsearchPillow_{}_reindexer".format(
DOMAIN_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(iteration_key, [Domain])
options = {'chunk_size': 5}
options.update(self.options)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
doc_transform=transform_domain_for_elasticsearch,
pillow=get_domain_kafka_to_elasticsearch_pillow(),
**options)
def build(self):
iteration_key = "UserToElasticsearchPillow_{}_reindexer".format(
USER_INDEX)
doc_provider = CouchDocumentProvider(iteration_key,
[CommCareUser, WebUser])
options = {'chunk_size': 5}
options.update(self.options)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_transform=transform_user_for_elasticsearch,
pillow=get_user_pillow_old(),
**options)
def build(self):
iteration_key = "ApplicationToElasticsearchPillow_{}_reindexer".format(
APP_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(
iteration_key, [Application, RemoteApp, LinkedApplication])
options = {'chunk_size': 5}
options.update(self.options)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_transform=transform_app_for_es,
pillow=get_app_to_elasticsearch_pillow(),
**options)
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain', None)
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_INDEX_INFO.index, limit_to_db or 'all', domain or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
reindex_accessor = CaseReindexAccessor(
domain=domain, limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain')
if not domain_needs_search_index(domain):
raise CaseSearchNotEnabledException(
"{} does not have case search enabled".format(domain))
iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain
or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
accessor = CaseReindexAccessor(domain=domain,
limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_SEARCH_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def get_couch_form_reindexer():
iteration_key = "CouchXFormToElasticsearchPillow_{}_reindexer".format(XFORM_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[
XFormInstance,
XFormArchived,
XFormError,
XFormDeprecated,
XFormDuplicate,
('XFormInstance-Deleted', XFormInstance),
('HQSubmission', XFormInstance),
SubmissionErrorLog,
])
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_filter=xform_pillow_filter,
doc_transform=transform_xform_for_elasticsearch,
pillow=get_xform_to_elasticsearch_pillow(),
)
