def get_document_provider(self, date_range=None):
iteration_key = self.iteration_key
if date_range:
(start, end) = date_range
self.reindexer.date_range = date_range
iteration_key = '{}-{}-{}'.format(self.iteration_key, start, end)
return SqlDocumentProvider(iteration_key, self.reindexer)
def build(self):
iteration_key = "SqlAppFormSubmissionTrackerPillow_reindexer"
doc_provider = SqlDocumentProvider(
iteration_key, FormReindexAccessor(include_attachments=False))
return AppFormSubmissionReindexer(doc_provider, FORM_SQL,
'form_processor_xforminstancesql',
**self.options)
def get_sql_case_reindexer():
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format(
CASE_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, CaseReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch)
def get_ledger_v2_reindexer():
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format(
LEDGER_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, LedgerReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=LEDGER_INDEX_INFO,
doc_transform=_prepare_ledger_for_es)
def get_sql_form_reindexer():
iteration_key = "SqlXFormToElasticsearchPillow_{}_reindexer".format(
XFORM_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, FormReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_transform=transform_xform_for_elasticsearch)
def build(self):
iteration_key = f"SmsToElasticsearchPillow_{SMS_INDEX_INFO.index}_reindexer"
reindex_accessor = SMSReindexAccessor()
doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor)
return ResumableBulkElasticPillowReindexer(doc_provider,
elasticsearch=get_es_new(),
index_info=SMS_INDEX_INFO,
pillow=get_sql_sms_pillow(),
**self.options)
def run(self, iteration_key):
"""
iterate sql records and update them as and when needed
"""
record_provider = SqlDocumentProvider(iteration_key,
self.case_accessor())
processor = BulkDocProcessor(record_provider,
self.doc_processor(self.domain))
return processor.run()
def get_document_provider(self, date_range=None, domain=None):
iteration_key = self.iteration_key
if date_range:
(start, end) = date_range
self.reindexer.date_range = date_range
iteration_key = f'{iteration_key}-{start}-{end}'
if domain:
self.reindexer.domain = domain
iteration_key = f'{iteration_key}-{domain}'
return SqlDocumentProvider(iteration_key, self.reindexer)
def build(self):
domain = self.options.pop('domain', None)
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}".format(
LEDGER_INDEX_INFO.index, domain or 'all')
doc_provider = SqlDocumentProvider(
iteration_key, LedgerReindexAccessor(domain=domain))
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=LEDGER_INDEX_INFO,
doc_transform=_prepare_ledger_for_es,
**self.options)
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain', None)
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_INDEX_INFO.index, limit_to_db or 'all', domain or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
reindex_accessor = CaseReindexAccessor(
domain=domain, limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100):
if not self.domain:
raise MigrationError("Must specify domain")
doc_migrator = self.doc_migrator_class(self.slug, self.domain)
document_provider = SqlDocumentProvider(self.iteration_key,
FormReindexAccessor())
processor = DocumentProcessorController(
document_provider,
doc_migrator,
reset,
max_retry,
chunk_size,
)
return processor.run()
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain')
if not domain_needs_search_index(domain):
raise CaseSearchNotEnabledException(
"{} does not have case search enabled".format(domain))
iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain
or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
accessor = CaseReindexAccessor(domain=domain,
limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_SEARCH_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def get_document_provider(self):
return SqlDocumentProvider(self.iteration_key, self.reindexer)
def get_sql_app_form_submission_tracker_reindexer():
iteration_key = "SqlAppFormSubmissionTrackerPillow_reindexer"
doc_provider = SqlDocumentProvider(
iteration_key, FormReindexAccessor(include_attachments=False))
return AppFormSubmissionReindexer(doc_provider, FORM_SQL,
'form_processor_xforminstancesql')