def get_document_provider(self, date_range=None): iteration_key = self.iteration_key if date_range: (start, end) = date_range self.reindexer.date_range = date_range iteration_key = '{}-{}-{}'.format(self.iteration_key, start, end) return SqlDocumentProvider(iteration_key, self.reindexer)
def build(self): iteration_key = "SqlAppFormSubmissionTrackerPillow_reindexer" doc_provider = SqlDocumentProvider( iteration_key, FormReindexAccessor(include_attachments=False)) return AppFormSubmissionReindexer(doc_provider, FORM_SQL, 'form_processor_xforminstancesql', **self.options)
def get_sql_case_reindexer(): iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format( CASE_INDEX_INFO.index) doc_provider = SqlDocumentProvider(iteration_key, CaseReindexAccessor()) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_transform=transform_case_for_elasticsearch)
def get_ledger_v2_reindexer(): iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format( LEDGER_INDEX_INFO.index) doc_provider = SqlDocumentProvider(iteration_key, LedgerReindexAccessor()) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=LEDGER_INDEX_INFO, doc_transform=_prepare_ledger_for_es)
def get_sql_form_reindexer(): iteration_key = "SqlXFormToElasticsearchPillow_{}_reindexer".format( XFORM_INDEX_INFO.index) doc_provider = SqlDocumentProvider(iteration_key, FormReindexAccessor()) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=XFORM_INDEX_INFO, doc_transform=transform_xform_for_elasticsearch)
def build(self): iteration_key = f"SmsToElasticsearchPillow_{SMS_INDEX_INFO.index}_reindexer" reindex_accessor = SMSReindexAccessor() doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor) return ResumableBulkElasticPillowReindexer(doc_provider, elasticsearch=get_es_new(), index_info=SMS_INDEX_INFO, pillow=get_sql_sms_pillow(), **self.options)
def run(self, iteration_key): """ iterate sql records and update them as and when needed """ record_provider = SqlDocumentProvider(iteration_key, self.case_accessor()) processor = BulkDocProcessor(record_provider, self.doc_processor(self.domain)) return processor.run()
def get_document_provider(self, date_range=None, domain=None): iteration_key = self.iteration_key if date_range: (start, end) = date_range self.reindexer.date_range = date_range iteration_key = f'{iteration_key}-{start}-{end}' if domain: self.reindexer.domain = domain iteration_key = f'{iteration_key}-{domain}' return SqlDocumentProvider(iteration_key, self.reindexer)
def build(self): domain = self.options.pop('domain', None) iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}".format( LEDGER_INDEX_INFO.index, domain or 'all') doc_provider = SqlDocumentProvider( iteration_key, LedgerReindexAccessor(domain=domain)) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=LEDGER_INDEX_INFO, doc_transform=_prepare_ledger_for_es, **self.options)
def build(self): limit_to_db = self.options.pop('limit_to_db', None) domain = self.options.pop('domain', None) iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}_{}".format( CASE_INDEX_INFO.index, limit_to_db or 'all', domain or 'all') limit_db_aliases = [limit_to_db] if limit_to_db else None reindex_accessor = CaseReindexAccessor( domain=domain, limit_db_aliases=limit_db_aliases) doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_transform=transform_case_for_elasticsearch, **self.options)
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100): if not self.domain: raise MigrationError("Must specify domain") doc_migrator = self.doc_migrator_class(self.slug, self.domain) document_provider = SqlDocumentProvider(self.iteration_key, FormReindexAccessor()) processor = DocumentProcessorController( document_provider, doc_migrator, reset, max_retry, chunk_size, ) return processor.run()
def build(self): limit_to_db = self.options.pop('limit_to_db', None) domain = self.options.pop('domain') if not domain_needs_search_index(domain): raise CaseSearchNotEnabledException( "{} does not have case search enabled".format(domain)) iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format( CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain or 'all') limit_db_aliases = [limit_to_db] if limit_to_db else None accessor = CaseReindexAccessor(domain=domain, limit_db_aliases=limit_db_aliases) doc_provider = SqlDocumentProvider(iteration_key, accessor) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_SEARCH_INDEX_INFO, doc_transform=transform_case_for_elasticsearch, **self.options)
def get_document_provider(self): return SqlDocumentProvider(self.iteration_key, self.reindexer)
def get_sql_app_form_submission_tracker_reindexer(): iteration_key = "SqlAppFormSubmissionTrackerPillow_reindexer" doc_provider = SqlDocumentProvider( iteration_key, FormReindexAccessor(include_attachments=False)) return AppFormSubmissionReindexer(doc_provider, FORM_SQL, 'form_processor_xforminstancesql')