def discard_migration_state(slug):
migrator = mod.MIGRATIONS[slug]
iterator = CouchDocumentProvider(
migrator.iteration_key, migrator.doc_types
).get_document_iterator(1)
iterator.discard_state()
mod.BlobMigrationState.objects.filter(slug=slug).delete()
def discard_migration_state(slug):
migrator = mod.MIGRATIONS[slug]
iterator = CouchDocumentProvider(
migrator.iteration_key,
migrator.doc_types).get_document_iterator(1)
iterator.discard_state()
mod.BlobMigrationState.objects.filter(slug=slug).delete()
def build(self):
iteration_key = "UpdateUserSyncHistoryPillow_reindexer"
doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[
CommCareUser,
WebUser
])
return UserSyncHistoryReindexer(doc_provider, **self.options)
def build(self):
iteration_key = "UserAppFormSubmissionTrackerPillow_reindexer"
doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[
CommCareUser,
WebUser
])
return UserAppFormSubmissionReindexer(doc_provider, **self.options)
def get_app_reindexer():
iteration_key = "ApplicationToElasticsearchPillow_{}_reindexer".format(APP_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(iteration_key, [Application, RemoteApp])
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_transform=transform_app_for_es,
pillow=get_app_to_elasticsearch_pillow(),
)
def get_couch_case_reindexer():
iteration_key = "CouchCaseToElasticsearchPillow_{}_reindexer".format(
CASE_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(
iteration_key,
doc_type_tuples=[CommCareCase, ("CommCareCase-Deleted", CommCareCase)])
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
pillow=get_case_to_elasticsearch_pillow())
def build(self):
iteration_key = "DomainToElasticsearchPillow_{}_reindexer".format(
DOMAIN_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(iteration_key, [Domain])
options = {'chunk_size': 5}
options.update(self.options)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
doc_transform=transform_domain_for_elasticsearch,
pillow=get_domain_kafka_to_elasticsearch_pillow(),
**options)
def build(self):
iteration_key = "ApplicationToElasticsearchPillow_{}_reindexer".format(
APP_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(
iteration_key, [Application, RemoteApp, LinkedApplication])
options = {'chunk_size': 5}
options.update(self.options)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=APP_INDEX_INFO,
doc_transform=transform_app_for_es,
pillow=get_app_to_elasticsearch_pillow(),
**options)
def get_couch_app_form_submission_tracker_reindexer():
iteration_key = "CouchAppFormSubmissionTrackerPillow_reindexer"
doc_provider = CouchDocumentProvider(iteration_key,
doc_type_tuples=[
XFormInstance,
XFormArchived,
XFormError,
XFormDeprecated,
XFormDuplicate,
('HQSubmission', XFormInstance),
SubmissionErrorLog,
])
return AppFormSubmissionReindexer(doc_provider, COUCH,
XFormInstance.get_db().dbname)
def build(self):
iteration_key = "UserToElasticsearchPillow_{}_reindexer".format(
USER_INDEX)
doc_provider = CouchDocumentProvider(iteration_key,
[CommCareUser, WebUser])
options = {'chunk_size': 5}
options.update(self.options)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
doc_transform=transform_user_for_elasticsearch,
pillow=get_user_pillow_old(),
**options)
def build(self):
iteration_key = "CouchAppFormSubmissionTrackerPillow_reindexer"
doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[
XFormInstance,
XFormArchived,
XFormError,
XFormDeprecated,
XFormDuplicate,
('HQSubmission', XFormInstance),
SubmissionErrorLog,
])
return AppFormSubmissionReindexer(
doc_provider, SOURCE_COUCH, XFormInstance.get_db().dbname, **self.options
)
def _get_processor(self, chunk_size=2, ignore_docs=None, skip_docs=None, reset=False, doc_types=None):
doc_types = doc_types or [Bar]
doc_processor = DemoProcessor()
doc_provider = CouchDocumentProvider(self.processor_slug, doc_types)
processor = self.processor_class(
doc_provider,
doc_processor,
chunk_size=chunk_size,
reset=reset
)
processor.document_iterator.couch_db = self.db
if ignore_docs:
doc_processor.ignore_docs = ignore_docs
if skip_docs:
doc_processor.skip_docs = skip_docs
return doc_processor, processor
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100):
doc_migrator = self.doc_migrator_class(self.slug, self.couchdb,
filename)
progress_logger = CouchProcessorProgressLogger(self.doc_types)
document_provider = CouchDocumentProvider(self.iteration_key,
self.doc_types)
processor = DocumentProcessorController(
document_provider,
doc_migrator,
reset,
max_retry,
chunk_size,
progress_logger=progress_logger)
return processor.run()
def get_couch_form_reindexer():
iteration_key = "CouchXFormToElasticsearchPillow_{}_reindexer".format(XFORM_INDEX_INFO.index)
doc_provider = CouchDocumentProvider(iteration_key, doc_type_tuples=[
XFormInstance,
XFormArchived,
XFormError,
XFormDeprecated,
XFormDuplicate,
('XFormInstance-Deleted', XFormInstance),
('HQSubmission', XFormInstance),
SubmissionErrorLog,
])
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_filter=xform_pillow_filter,
doc_transform=transform_xform_for_elasticsearch,
pillow=get_xform_to_elasticsearch_pillow(),
)
def get_user_form_submission_tracker_reindexer():
iteration_key = "UserAppFormSubmissionTrackerPillow_reindexer"
doc_provider = CouchDocumentProvider(
iteration_key, doc_type_tuples=[CommCareUser, WebUser])
return UserAppFormSubmissionReindexer(doc_provider)
def get_document_provider(self):
return CouchDocumentProvider(self.iteration_key, self.doc_types)
def get_user_sync_history_reindexer():
iteration_key = "UpdateUserSyncHistoryPillow_reindexer"
doc_provider = CouchDocumentProvider(
iteration_key, doc_type_tuples=[CommCareUser, WebUser])
return UserSyncHistoryReindexer(doc_provider)
