def _update_forms_in_db(db_name):
reindex_accessor = FormReindexAccessor(limit_db_aliases=[db_name])
doc_count = reindex_accessor.get_approximate_doc_count(db_name)
chunks = iter_all_ids_chunked(reindex_accessor)
processed = 0
for form_ids in chunks:
processed += len(form_ids)
_update_forms(db_name, tuple(form_ids))
if processed % 5000 == 0:
print('[progress] [%s] %s of %s' % (db_name, processed, doc_count))
print('[progress] [%s] Complete' % db_name)
def _update_forms_in_db(db_name):
reindex_accessor = FormReindexAccessor(limit_db_aliases=[db_name])
doc_count = reindex_accessor.get_approximate_doc_count(db_name)
chunks = iter_all_ids_chunked(reindex_accessor)
processed = 0
for form_ids in chunks:
processed += len(form_ids)
_update_forms(db_name, tuple(form_ids))
if processed % 5000 == 0:
print('[progress] [%s] %s of %s' % (db_name, processed, doc_count))
print('[progress] [%s] Complete' % db_name)
def build(self):
iteration_key = "SqlAppFormSubmissionTrackerPillow_reindexer"
doc_provider = SqlDocumentProvider(
iteration_key, FormReindexAccessor(include_attachments=False))
return AppFormSubmissionReindexer(doc_provider, FORM_SQL,
'form_processor_xforminstancesql',
**self.options)
def get_sql_form_reindexer():
iteration_key = "SqlXFormToElasticsearchPillow_{}_reindexer".format(
XFORM_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, FormReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_transform=transform_xform_for_elasticsearch)
def handle(self, **options):
csv_mode = options['csv']
form_info = _get_counts(FormReindexAccessor())
case_info = _get_counts(CaseReindexAccessor())
if csv_mode:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([20, 20, 10])
_write_info('FORMS', form_info, row_formatter, self.stdout)
_write_info('CASES', case_info, row_formatter, self.stdout)
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100):
if not self.domain:
raise MigrationError("Must specify domain")
doc_migrator = self.doc_migrator_class(self.slug, self.domain)
document_provider = SqlDocumentProvider(self.iteration_key,
FormReindexAccessor())
processor = DocumentProcessorController(
document_provider,
doc_migrator,
reset,
max_retry,
chunk_size,
)
return processor.run()
def check_for_sql_forms_without_existing_domain():
missing_domains_with_forms = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
accessor = FormReindexAccessor(domain=domain, include_deleted=True)
for db_name in accessor.sql_db_aliases:
if _has_docs(accessor, db_name):
missing_domains_with_forms |= {domain}
break
if missing_domains_with_forms:
mail_admins_async.delay(
'There exist SQL forms belonging to a missing domain',
str(missing_domains_with_forms))
elif _is_monday():
mail_admins_async.delay('All SQL forms belong to valid domains', '')
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain', None)
iteration_key = "SqlXFormToElasticsearchPillow_{}_reindexer_{}_{}".format(
XFORM_INDEX_INFO.index, limit_to_db or 'all', domain or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
reindex_accessor = FormReindexAccessor(
domain=domain, limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=XFORM_INDEX_INFO,
doc_filter=xform_pillow_filter,
doc_transform=transform_xform_for_elasticsearch,
**self.options)
def get_sql_chunks(run_config):
domain = run_config.domain if run_config.domain is not ALL_DOMAINS else None
accessor = FormReindexAccessor(
domain,
start_date=run_config.start_date,
end_date=run_config.end_date,
)
iteration_key = f'sql_forms-{run_config.iteration_key}'
for chunk in _get_resumable_chunked_iterator(accessor, iteration_key,
'[SQL forms] '):
matching_records = [
(form.form_id, form.doc_type, form.xmlns, form.received_on,
form.domain) for form in chunk
# Only check for "normal" and "archived" forms
if form.is_normal or form.is_archived
]
yield matching_records
def reindex_accessor(self):
return FormReindexAccessor()
def get_sql_app_form_submission_tracker_reindexer():
iteration_key = "SqlAppFormSubmissionTrackerPillow_reindexer"
doc_provider = SqlDocumentProvider(
iteration_key, FormReindexAccessor(include_attachments=False))
return AppFormSubmissionReindexer(doc_provider, FORM_SQL,
'form_processor_xforminstancesql')
