def discard_migration_state(slug): migrator = mod.MIGRATIONS[slug] iterator = CouchDocumentProvider( migrator.iteration_key, migrator.doc_types ).get_document_iterator(1) iterator.discard_state() mod.BlobMigrationState.objects.filter(slug=slug).delete()
def discard_migration_state(slug): migrator = mod.MIGRATIONS[slug] iterator = CouchDocumentProvider( migrator.iteration_key, migrator.doc_types).get_document_iterator(1) iterator.discard_state() mod.BlobMigrationState.objects.filter(slug=slug).delete()
def build(self): iteration_key = "UpdateUserSyncHistoryPillow_reindexer" doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[ CommCareUser, WebUser ]) return UserSyncHistoryReindexer(doc_provider, **self.options)
def build(self): iteration_key = "UserAppFormSubmissionTrackerPillow_reindexer" doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[ CommCareUser, WebUser ]) return UserAppFormSubmissionReindexer(doc_provider, **self.options)
def get_app_reindexer(): iteration_key = "ApplicationToElasticsearchPillow_{}_reindexer".format(APP_INDEX_INFO.index) doc_provider = CouchDocumentProvider(iteration_key, [Application, RemoteApp]) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=APP_INDEX_INFO, doc_transform=transform_app_for_es, pillow=get_app_to_elasticsearch_pillow(), )
def get_couch_case_reindexer(): iteration_key = "CouchCaseToElasticsearchPillow_{}_reindexer".format( CASE_INDEX_INFO.index) doc_provider = CouchDocumentProvider( iteration_key, doc_type_tuples=[CommCareCase, ("CommCareCase-Deleted", CommCareCase)]) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_transform=transform_case_for_elasticsearch, pillow=get_case_to_elasticsearch_pillow())
def build(self): iteration_key = "DomainToElasticsearchPillow_{}_reindexer".format( DOMAIN_INDEX_INFO.index) doc_provider = CouchDocumentProvider(iteration_key, [Domain]) options = {'chunk_size': 5} options.update(self.options) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=DOMAIN_INDEX_INFO, doc_transform=transform_domain_for_elasticsearch, pillow=get_domain_kafka_to_elasticsearch_pillow(), **options)
def build(self): iteration_key = "ApplicationToElasticsearchPillow_{}_reindexer".format( APP_INDEX_INFO.index) doc_provider = CouchDocumentProvider( iteration_key, [Application, RemoteApp, LinkedApplication]) options = {'chunk_size': 5} options.update(self.options) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=APP_INDEX_INFO, doc_transform=transform_app_for_es, pillow=get_app_to_elasticsearch_pillow(), **options)
def get_couch_app_form_submission_tracker_reindexer(): iteration_key = "CouchAppFormSubmissionTrackerPillow_reindexer" doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[ XFormInstance, XFormArchived, XFormError, XFormDeprecated, XFormDuplicate, ('HQSubmission', XFormInstance), SubmissionErrorLog, ]) return AppFormSubmissionReindexer(doc_provider, COUCH, XFormInstance.get_db().dbname)
def build(self): iteration_key = "UserToElasticsearchPillow_{}_reindexer".format( USER_INDEX) doc_provider = CouchDocumentProvider(iteration_key, [CommCareUser, WebUser]) options = {'chunk_size': 5} options.update(self.options) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=USER_INDEX_INFO, doc_transform=transform_user_for_elasticsearch, pillow=get_user_pillow_old(), **options)
def build(self): iteration_key = "CouchAppFormSubmissionTrackerPillow_reindexer" doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[ XFormInstance, XFormArchived, XFormError, XFormDeprecated, XFormDuplicate, ('HQSubmission', XFormInstance), SubmissionErrorLog, ]) return AppFormSubmissionReindexer( doc_provider, SOURCE_COUCH, XFormInstance.get_db().dbname, **self.options )
def _get_processor(self, chunk_size=2, ignore_docs=None, skip_docs=None, reset=False, doc_types=None): doc_types = doc_types or [Bar] doc_processor = DemoProcessor() doc_provider = CouchDocumentProvider(self.processor_slug, doc_types) processor = self.processor_class( doc_provider, doc_processor, chunk_size=chunk_size, reset=reset ) processor.document_iterator.couch_db = self.db if ignore_docs: doc_processor.ignore_docs = ignore_docs if skip_docs: doc_processor.skip_docs = skip_docs return doc_processor, processor
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100): doc_migrator = self.doc_migrator_class(self.slug, self.couchdb, filename) progress_logger = CouchProcessorProgressLogger(self.doc_types) document_provider = CouchDocumentProvider(self.iteration_key, self.doc_types) processor = DocumentProcessorController( document_provider, doc_migrator, reset, max_retry, chunk_size, progress_logger=progress_logger) return processor.run()
def get_couch_form_reindexer(): iteration_key = "CouchXFormToElasticsearchPillow_{}_reindexer".format(XFORM_INDEX_INFO.index) doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[ XFormInstance, XFormArchived, XFormError, XFormDeprecated, XFormDuplicate, ('XFormInstance-Deleted', XFormInstance), ('HQSubmission', XFormInstance), SubmissionErrorLog, ]) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=XFORM_INDEX_INFO, doc_filter=xform_pillow_filter, doc_transform=transform_xform_for_elasticsearch, pillow=get_xform_to_elasticsearch_pillow(), )
def get_user_form_submission_tracker_reindexer(): iteration_key = "UserAppFormSubmissionTrackerPillow_reindexer" doc_provider = CouchDocumentProvider( iteration_key, doc_type_tuples=[CommCareUser, WebUser]) return UserAppFormSubmissionReindexer(doc_provider)
def get_document_provider(self): return CouchDocumentProvider(self.iteration_key, self.doc_types)
def get_user_sync_history_reindexer(): iteration_key = "UpdateUserSyncHistoryPillow_reindexer" doc_provider = CouchDocumentProvider( iteration_key, doc_type_tuples=[CommCareUser, WebUser]) return UserSyncHistoryReindexer(doc_provider)