def iter_all_changes(self, start_from=None): accessor = CaseReindexAccessor(self.domain, limit_db_aliases=self.limit_db_aliases) for db_alias in accessor.sql_db_aliases: cases = accessor.get_docs(db_alias, start_from) while cases: for case in cases: yield _sql_case_to_change(case) start_from_for_db = case.server_modified_on last_id = case.id cases = accessor.get_docs(db_alias, start_from_for_db, last_doc_pk=last_id)
def _find_case_ids_with_invalid_phone_number(self): case_ids_with_invalid_phone_number = [] start_date = date.today() - timedelta(days=100) reindex_accessor = CaseReindexAccessor( domain=DOMAIN, case_type='person', limit_db_aliases=[self.db_alias], start_date=start_date) filename = 'invalid_phone_numbers_with_91_part_%s_%s.csv' % ( self.db_alias, datetime.utcnow()) with open(filename, 'w') as output: cases_iterated = 0 writer = csv.writer(output) writer.writerow(CSV_HEADERS) if self.log_progress: self.stdout.write('iterating now') for case in iter_all_rows(reindex_accessor): if self.log_progress and cases_iterated % CASE_ITERATION_COUNT == 0: self.stdout.write("cases iterated: %s" % cases_iterated) cases_iterated += 1 if self._case_needs_to_be_updated(case): case_ids_with_invalid_phone_number.append(case.case_id) writer.writerow([case.case_id])
def _find_case_ids_without_mother_name(self): start_date = date.today() - timedelta(days=100) reindex_accessor = CaseReindexAccessor( domain=DOMAIN, case_type=CASE_TYPE, limit_db_aliases=[self.db_alias], start_date=start_date) filename = 'cases_without_mother_name_part_%s_%s.csv' % ( self.db_alias, datetime.utcnow()) cases_with_no_mother_name_filename = 'cases_with_no_mother_name_part_%s_%s.csv' % ( self.db_alias, datetime.utcnow()) with open(filename, 'w') as output: with open(cases_with_no_mother_name_filename, 'w') as no_mother_name_file: cases_iterated = 0 writer = csv.writer(output) writer.writerow(CSV_HEADERS) no_mother_name_writer = csv.writer(no_mother_name_file) no_mother_name_writer.writerow(['Case ID']) if self.log_progress: print('iterating now') for case in iter_all_rows(reindex_accessor): if self.log_progress and cases_iterated % CASE_ITERATION_COUNT == 0: print("cases iterated: %s" % cases_iterated) cases_iterated += 1 if self._case_needs_to_be_updated(case): mother_case_id, mother_name = self._find_mother_case_id_and_name( case) if mother_case_id and mother_name: writer.writerow( [case.case_id, mother_case_id, mother_name]) else: no_mother_name_writer.writerow([case.case_id])
def iter_document_ids(self): if should_use_sql_backend(self.domain): accessor = CaseReindexAccessor(self.domain, case_type=self.case_type) return iter_all_ids(accessor) else: return iter( self.case_accessors.get_case_ids_in_domain(self.case_type))
def get_sql_case_reindexer(): iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer".format( CASE_INDEX_INFO.index) doc_provider = SqlDocumentProvider(iteration_key, CaseReindexAccessor()) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_transform=transform_case_for_elasticsearch)
def handle(self, **options): csv_mode = options['csv'] form_info = _get_counts(FormReindexAccessor()) case_info = _get_counts(CaseReindexAccessor()) if csv_mode: row_formatter = CSVRowFormatter() else: row_formatter = TableRowFormatter([20, 20, 10]) _write_info('FORMS', form_info, row_formatter, self.stdout) _write_info('CASES', case_info, row_formatter, self.stdout)
def get_sql_chunks(run_config): domain = run_config.domain if run_config.domain is not ALL_DOMAINS else None accessor = CaseReindexAccessor(domain, start_date=run_config.start_date, end_date=run_config.end_date, case_type=run_config.case_type) iteration_key = f'sql_cases-{run_config.iteration_key}' for chunk in _get_resumable_chunked_iterator(accessor, iteration_key, '[SQL cases] '): matching_records = [(case.case_id, case.type, case.server_modified_on, case.domain) for case in chunk] yield matching_records
def build(self): limit_to_db = self.options.pop('limit_to_db', None) domain = self.options.pop('domain', None) iteration_key = "SqlCaseToElasticsearchPillow_{}_reindexer_{}_{}".format( CASE_INDEX_INFO.index, limit_to_db or 'all', domain or 'all') limit_db_aliases = [limit_to_db] if limit_to_db else None reindex_accessor = CaseReindexAccessor( domain=domain, limit_db_aliases=limit_db_aliases) doc_provider = SqlDocumentProvider(iteration_key, reindex_accessor) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_INDEX_INFO, doc_transform=transform_case_for_elasticsearch, **self.options)
def check_for_sql_cases_without_existing_domain(): missing_domains_with_cases = set() for domain in set( _get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names()): accessor = CaseReindexAccessor(domain=domain, include_deleted=True) for db_name in accessor.sql_db_aliases: if _has_docs(accessor, db_name): missing_domains_with_cases |= {domain} break if missing_domains_with_cases: mail_admins_async.delay( 'There exist SQL cases belonging to a missing domain', str(missing_domains_with_cases)) elif _is_monday(): mail_admins_async.delay('All SQL cases belong to valid domains', '')
def sync_cases(self, domain): db_aliases = get_db_aliases_for_partitioned_query() db_aliases.sort() if should_use_sql_backend(domain): case_accessor = CaseReindexAccessor(domain) case_ids = (case.case_id for case in iter_all_rows(case_accessor)) else: changes = _iter_changes(domain, CASE_DOC_TYPES) case_ids = (case.id for case in changes) next_event = time.time() + 10 for i, case_id in enumerate(case_ids): sync_case_for_messaging.delay(domain, case_id) if time.time() > next_event: print("Queued %d cases for domain %s" % (i + 1, domain)) next_event = time.time() + 10
def build(self): limit_to_db = self.options.pop('limit_to_db', None) domain = self.options.pop('domain') if not domain_needs_search_index(domain): raise CaseSearchNotEnabledException( "{} does not have case search enabled".format(domain)) iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format( CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain or 'all') limit_db_aliases = [limit_to_db] if limit_to_db else None accessor = CaseReindexAccessor(domain=domain, limit_db_aliases=limit_db_aliases) doc_provider = SqlDocumentProvider(iteration_key, accessor) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_SEARCH_INDEX_INFO, doc_transform=transform_case_for_elasticsearch, **self.options)
def iter_document_ids(self): accessor = CaseReindexAccessor(self.domain, case_type=self.case_type) return iter_all_ids(accessor)
def reindex_accessor(self): return CaseReindexAccessor()
def perform_migration(domain, case_type, migration_xml): accessor = CaseReindexAccessor(domain=domain, case_type=case_type) for case_ids in iter_all_ids_chunked(accessor): send_migration_to_formplayer(domain, case_ids, migration_xml)
def iter_all_changes(self, start_from=None): accessor = CaseReindexAccessor(self.domain, limit_db_aliases=self.limit_db_aliases) for case in iter_all_rows(accessor): yield _sql_case_to_change(case)
def case_accessor(self): return CaseReindexAccessor(domain=self.domain, case_type=self.case_type, limit_db_aliases=self.db_aliases, start_date=self.start_date, end_date=self.end_date)