def iter_all_changes(self, start_from=None):
accessor = CaseReindexAccessor(self.domain,
limit_db_aliases=self.limit_db_aliases)
for db_alias in accessor.sql_db_aliases:
cases = accessor.get_docs(db_alias, start_from)
while cases:
for case in cases:
yield _sql_case_to_change(case)
start_from_for_db = case.server_modified_on
last_id = case.id
cases = accessor.get_docs(db_alias,
start_from_for_db,
last_doc_pk=last_id)
def _find_case_ids_with_invalid_phone_number(self):
case_ids_with_invalid_phone_number = []
start_date = date.today() - timedelta(days=100)
reindex_accessor = CaseReindexAccessor(
domain=DOMAIN,
case_type='person',
limit_db_aliases=[self.db_alias],
start_date=start_date)
filename = 'invalid_phone_numbers_with_91_part_%s_%s.csv' % (
self.db_alias, datetime.utcnow())
with open(filename, 'w') as output:
cases_iterated = 0
writer = csv.writer(output)
writer.writerow(CSV_HEADERS)
if self.log_progress:
self.stdout.write('iterating now')
for case in iter_all_rows(reindex_accessor):
if self.log_progress and cases_iterated % CASE_ITERATION_COUNT == 0:
self.stdout.write("cases iterated: %s" % cases_iterated)
cases_iterated += 1
if self._case_needs_to_be_updated(case):
case_ids_with_invalid_phone_number.append(case.case_id)
writer.writerow([case.case_id])
def _find_case_ids_without_mother_name(self):
start_date = date.today() - timedelta(days=100)
reindex_accessor = CaseReindexAccessor(
domain=DOMAIN,
case_type=CASE_TYPE,
limit_db_aliases=[self.db_alias],
start_date=start_date)
filename = 'cases_without_mother_name_part_%s_%s.csv' % (
self.db_alias, datetime.utcnow())
cases_with_no_mother_name_filename = 'cases_with_no_mother_name_part_%s_%s.csv' % (
self.db_alias, datetime.utcnow())
with open(filename, 'w') as output:
with open(cases_with_no_mother_name_filename,
'w') as no_mother_name_file:
cases_iterated = 0
writer = csv.writer(output)
writer.writerow(CSV_HEADERS)
no_mother_name_writer = csv.writer(no_mother_name_file)
no_mother_name_writer.writerow(['Case ID'])
if self.log_progress:
print('iterating now')
for case in iter_all_rows(reindex_accessor):
if self.log_progress and cases_iterated % CASE_ITERATION_COUNT == 0:
print("cases iterated: %s" % cases_iterated)
cases_iterated += 1
if self._case_needs_to_be_updated(case):
mother_case_id, mother_name = self._find_mother_case_id_and_name(
case)
if mother_case_id and mother_name:
writer.writerow(
[case.case_id, mother_case_id, mother_name])
else:
no_mother_name_writer.writerow([case.case_id])
def iter_document_ids(self):
if should_use_sql_backend(self.domain):
accessor = CaseReindexAccessor(self.domain,
case_type=self.case_type)
return iter_all_ids(accessor)
else:
return iter(
self.case_accessors.get_case_ids_in_domain(self.case_type))
def get_sql_case_reindexer():
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format(
CASE_INDEX_INFO.index)
doc_provider = SqlDocumentProvider(iteration_key, CaseReindexAccessor())
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch)
def handle(self, **options):
csv_mode = options['csv']
form_info = _get_counts(FormReindexAccessor())
case_info = _get_counts(CaseReindexAccessor())
if csv_mode:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([20, 20, 10])
_write_info('FORMS', form_info, row_formatter, self.stdout)
_write_info('CASES', case_info, row_formatter, self.stdout)
def get_sql_chunks(run_config):
domain = run_config.domain if run_config.domain is not ALL_DOMAINS else None
accessor = CaseReindexAccessor(domain,
start_date=run_config.start_date,
end_date=run_config.end_date,
case_type=run_config.case_type)
iteration_key = f'sql_cases-{run_config.iteration_key}'
for chunk in _get_resumable_chunked_iterator(accessor, iteration_key,
'[SQL cases] '):
matching_records = [(case.case_id, case.type,
case.server_modified_on, case.domain)
for case in chunk]
yield matching_records
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain', None)
iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_INDEX_INFO.index, limit_to_db or 'all', domain or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
reindex_accessor = CaseReindexAccessor(
domain=domain, limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def check_for_sql_cases_without_existing_domain():
missing_domains_with_cases = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
accessor = CaseReindexAccessor(domain=domain, include_deleted=True)
for db_name in accessor.sql_db_aliases:
if _has_docs(accessor, db_name):
missing_domains_with_cases |= {domain}
break
if missing_domains_with_cases:
mail_admins_async.delay(
'There exist SQL cases belonging to a missing domain',
str(missing_domains_with_cases))
elif _is_monday():
mail_admins_async.delay('All SQL cases belong to valid domains', '')
def sync_cases(self, domain):
db_aliases = get_db_aliases_for_partitioned_query()
db_aliases.sort()
if should_use_sql_backend(domain):
case_accessor = CaseReindexAccessor(domain)
case_ids = (case.case_id for case in iter_all_rows(case_accessor))
else:
changes = _iter_changes(domain, CASE_DOC_TYPES)
case_ids = (case.id for case in changes)
next_event = time.time() + 10
for i, case_id in enumerate(case_ids):
sync_case_for_messaging.delay(domain, case_id)
if time.time() > next_event:
print("Queued %d cases for domain %s" % (i + 1, domain))
next_event = time.time() + 10
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain')
if not domain_needs_search_index(domain):
raise CaseSearchNotEnabledException(
"{} does not have case search enabled".format(domain))
iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain
or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
accessor = CaseReindexAccessor(domain=domain,
limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_SEARCH_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def iter_document_ids(self):
accessor = CaseReindexAccessor(self.domain, case_type=self.case_type)
return iter_all_ids(accessor)
def reindex_accessor(self):
return CaseReindexAccessor()
def perform_migration(domain, case_type, migration_xml):
accessor = CaseReindexAccessor(domain=domain, case_type=case_type)
for case_ids in iter_all_ids_chunked(accessor):
send_migration_to_formplayer(domain, case_ids, migration_xml)
def iter_all_changes(self, start_from=None):
accessor = CaseReindexAccessor(self.domain,
limit_db_aliases=self.limit_db_aliases)
for case in iter_all_rows(accessor):
yield _sql_case_to_change(case)
def case_accessor(self):
return CaseReindexAccessor(domain=self.domain,
case_type=self.case_type,
limit_db_aliases=self.db_aliases,
start_date=self.start_date,
end_date=self.end_date)
