def test_no_checkpoint_creation(self, reindex_id, pillow_name):
# these pillows should not touch checkpoints since they are run with other
# reindexers
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
# set these to something obviously wrong
current_offsets = pillow.checkpoint.get_current_sequence_as_dict()
bad_offsets = {tp: (offset + 38014) for tp, offset in current_offsets.items()}
pillow.checkpoint.update_to(bad_offsets)
self.assertNotEqual(current_offsets, pillow.checkpoint.get_current_sequence_as_dict())
self.assertEqual(bad_offsets, pillow.checkpoint.get_current_sequence_as_dict())
reindex_and_clean(reindex_id)
# make sure they are still bad
pillow = get_pillow_by_name(pillow_name)
self.assertNotEqual(
current_offsets,
pillow.checkpoint.get_current_sequence_as_dict(),
)
self.assertEqual(
bad_offsets,
pillow.checkpoint.get_current_sequence_as_dict(),
)
self.assertNotEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_or_create_wrapped().wrapped_sequence,
)
self.assertNotEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_current_sequence_as_dict(),
)
def test_checkpoint_creation(self, reindex_id, pillow_name):
# checks that checkpoipnts are set to the latest checkpoints after reindexing
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
# set the offets to something obviously wrong
current_offsets = pillow.checkpoint.get_current_sequence_as_dict()
bad_offsets = {
tp: (offset + 38014)
for tp, offset in current_offsets.items()
}
pillow.checkpoint.update_to(bad_offsets)
self.assertNotEqual(current_offsets,
pillow.checkpoint.get_current_sequence_as_dict())
self.assertEqual(bad_offsets,
pillow.checkpoint.get_current_sequence_as_dict())
reindex_and_clean(reindex_id)
pillow = get_pillow_by_name(pillow_name)
self.assertNotEqual(bad_offsets,
pillow.checkpoint.get_current_sequence_as_dict())
self.assertEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_or_create_wrapped().wrapped_sequence,
)
self.assertEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_current_sequence_as_dict(),
)
def test_case_reindexer_v2(self):
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
index_id = 'sql-case' if settings.TESTS_SHOULD_USE_SQL_BACKEND else 'case'
reindex_and_clean(index_id, reset=True)
self._assert_case_is_in_es(case)
def test_xform_reindexer_v2(self):
FormProcessorTestUtils.delete_all_xforms()
form = create_and_save_a_form(DOMAIN)
index_id = 'sql-form' if settings.TESTS_SHOULD_USE_SQL_BACKEND else 'form'
reindex_and_clean(index_id, reset=True)
self._assert_form_is_in_es(form)
def test_case_reindexer_v2(self):
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
index_id = 'sql-case'
reindex_and_clean(index_id, reset=True)
self._assert_case_is_in_es(case)
def test_xform_reindexer_v2(self):
FormProcessorTestUtils.delete_all_xforms()
form = create_and_save_a_form(DOMAIN)
index_id = 'sql-form'
reindex_and_clean(index_id, reset=True)
self._assert_form_is_in_es(form)
def test_domain_reindexer(self):
delete_all_domains()
ensure_index_deleted(DOMAIN_INDEX)
name = 'reindex-test-domain'
create_domain(name)
reindex_and_clean('domain')
results = DomainES().run()
self.assertEqual(1, results.total, results.hits)
domain_doc = results.hits[0]
self.assertEqual(name, domain_doc['name'])
self.assertEqual('Domain', domain_doc['doc_type'])
delete_es_index(DOMAIN_INDEX)
def test_groups_to_user_reindexer(self):
initialize_index_and_mapping(self.es, USER_INDEX_INFO)
user_id = uuid.uuid4().hex
domain = 'test-groups-to-user-reindex'
_create_es_user(self.es, user_id, domain)
# create and save a group
group = Group(domain=domain, name='g1', users=[user_id])
group.save()
reindex_and_clean('groups-to-user')
self.es.indices.refresh(USER_INDEX)
_assert_es_user_and_groups(self, self.es, user_id, [group._id], [group.name])
class ReportCaseReindexerTest(TestCase):
def setUp(self):
super(ReportCaseReindexerTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
FormProcessorTestUtils.delete_all_cases()
with trap_extra_setup(ConnectionError):
self.elasticsearch = get_es_new()
ensure_index_deleted(REPORT_CASE_INDEX_INFO.index)
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms()
FormProcessorTestUtils.delete_all_cases()
ensure_index_deleted(REPORT_CASE_INDEX_INFO.index)
super(ReportCaseReindexerTest, self).tearDown()
@run_with_all_backends
def test_report_case_reindexer(self):
cases_included = set()
for i in range(3):
case = create_and_save_a_case(DOMAIN,
uuid.uuid4().hex,
'case_name-{}'.format(i))
cases_included.add(case.case_id)
# excluded case
create_and_save_a_case('unsupported', uuid.uuid4().hex, 'unsupported')
reindex_and_clean('report-case')
# verify there
results = CaseES("report_cases").run()
self.assertEqual(3, results.total, results.hits)
ids_in_es = {doc['_id'] for doc in results.hits}
self.assertEqual(cases_included, ids_in_es)
def test_report_case_reindexer(self):
cases_included = set()
for i in range(3):
case = create_and_save_a_case(DOMAIN, uuid.uuid4().hex, 'case_name-{}'.format(i))
cases_included.add(case.case_id)
# excluded case
create_and_save_a_case('unsupported', uuid.uuid4().hex, 'unsupported')
reindex_and_clean('report-case')
# verify there
results = CaseES("report_cases").run()
self.assertEqual(3, results.total, results.hits)
ids_in_es = {doc['_id'] for doc in results.hits}
self.assertEqual(cases_included, ids_in_es)
def test_ledger_reindexer(self):
factory = CaseFactory(domain=self.domain)
case = factory.create_case()
from corehq.apps.commtrack.tests.util import get_single_balance_block
from corehq.apps.hqcase.utils import submit_case_blocks
submit_case_blocks(
[get_single_balance_block(case.case_id, self.product_id, 100)],
self.domain)
ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id)
use_sql = should_use_sql_backend(self.domain)
index_id = 'ledger-v2' if use_sql else 'ledger-v1'
options = {'reset': True} if use_sql else {}
reindex_and_clean(index_id, **options)
self._assert_ledger_in_es(ref)
def test_case_search_reindexer(self):
es = get_es_new()
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
ensure_index_deleted(CASE_SEARCH_INDEX)
# With case search not enabled, case should not make it to ES
CaseSearchConfig.objects.all().delete()
reindex_and_clean('case-search')
es.indices.refresh(CASE_SEARCH_INDEX) # as well as refresh the index
self._assert_es_empty(esquery=CaseSearchES())
# With case search enabled, it should get indexed
CaseSearchConfig.objects.create(domain=self.domain, enabled=True)
self.addCleanup(CaseSearchConfig.objects.all().delete)
reindex_and_clean('case-search')
es.indices.refresh(CASE_SEARCH_INDEX) # as well as refresh the index
self._assert_case_is_in_es(case, esquery=CaseSearchES())
def test_case_search_reindexer(self):
es = get_es_new()
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
ensure_index_deleted(CASE_SEARCH_INDEX)
# With case search not enabled, case should not make it to ES
CaseSearchConfig.objects.all().delete()
domains_needing_search_index.clear()
reindex_and_clean('case-search')
es.indices.refresh(CASE_SEARCH_INDEX) # as well as refresh the index
self._assert_es_empty(esquery=CaseSearchES())
# With case search enabled, it should get indexed
with mock.patch('corehq.pillows.case_search.domains_needing_search_index',
mock.MagicMock(return_value=[self.domain])):
reindex_and_clean('case-search')
es.indices.refresh(CASE_SEARCH_INDEX) # as well as refresh the index
self._assert_case_is_in_es(case, esquery=CaseSearchES())
def test_group_reindexer(self):
group = Group(domain=DOMAIN, name='g1')
group.save()
reindex_and_clean('group')
self._assert_group_in_es(group)
def test_web_user_reindexer_v2(self):
username = '******'
WebUser.create(DOMAIN, username, 'secret', None, None)
reindex_and_clean('user')
self._assert_user_in_es(username, is_webuser=True)
def test_user_reindexer_v2(self):
username = '******'
CommCareUser.create(DOMAIN, username, 'secret', None, None)
reindex_and_clean('user')
self._assert_user_in_es(username)
