def test_case_search_reindex_by_domain(self):
"""
Tests reindexing for a particular domain only
"""
other_domain = "yunkai"
CaseSearchConfig.objects.get_or_create(pk=other_domain, enabled=True)
domains_needing_search_index.clear()
desired_case = self._make_case(domain=other_domain)
undesired_case = self._make_case(domain=self.domain) # noqa
with self.assertRaises(CaseSearchNotEnabledException):
CaseSearchReindexerFactory(domain=self.domain).build().reindex()
CaseSearchReindexerFactory(domain=other_domain).build().reindex()
self._assert_case_in_es(other_domain, desired_case)
def setUp(self):
self.domain = create_domain(DOMAIN)
self.user = CommCareUser.create(DOMAIN, USERNAME, PASSWORD, None, None)
initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO)
CaseSearchConfig.objects.get_or_create(pk=DOMAIN, enabled=True)
delete_all_cases()
self.case_id = uuid4().hex
_, [self.case] = post_case_blocks([
CaseBlock.deprecated_init(
create=True,
case_id=self.case_id,
case_type=CASE_TYPE,
case_name=CASE_NAME,
external_id=CASE_NAME,
user_id=OWNER_ID,
owner_id=OWNER_ID,
update={
'opened_by': OWNER_ID
},
).as_xml()
], {'domain': DOMAIN})
domains_needing_search_index.clear()
CaseSearchReindexerFactory(domain=DOMAIN).build().reindex()
es = get_es_new()
es.indices.refresh(CASE_SEARCH_INDEX)
def _bootstrap_cases_in_es_for_domain(self, domain, input_cases):
for case in input_cases:
index = case.pop('index', None)
self._make_case(domain, case, index=index)
with patch('corehq.pillows.case_search.domains_needing_search_index',
MagicMock(return_value=[domain])):
CaseSearchReindexerFactory(domain=domain).build().reindex()
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
def test_geopoint_property(self):
CaseSearchConfig.objects.get_or_create(pk=self.domain, enabled=True)
domains_needing_search_index.clear()
self._make_data_dictionary(
gps_properties=['coords', 'short_coords', 'other_coords'])
case = self._make_case(
case_properties={
'coords': '-33.8561 151.2152 0 0',
'short_coords': '-33.8561 151.2152',
'other_coords': '42 Wallaby Way',
'not_coords': '-33.8561 151.2152 0 0',
})
CaseSearchReindexerFactory(domain=self.domain).build().reindex()
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
es_case = CaseSearchES().doc_id(case.case_id).run().hits[0]
self.assertEqual(
self._get_prop(es_case['case_properties'], 'coords'),
{
'key': 'coords',
'value': '-33.8561 151.2152 0 0',
'geopoint_value': {
'lat': -33.8561,
'lon': 151.2152
},
},
)
self.assertEqual(
self._get_prop(es_case['case_properties'], 'short_coords'),
{
'key': 'short_coords',
'value': '-33.8561 151.2152',
'geopoint_value': {
'lat': -33.8561,
'lon': 151.2152
},
},
)
self.assertEqual(
self._get_prop(es_case['case_properties'], 'other_coords'),
# The value here isn't a valid geopoint
{
'key': 'other_coords',
'value': '42 Wallaby Way',
'geopoint_value': None
},
)
self.assertEqual(
self._get_prop(es_case['case_properties'], 'not_coords'),
# This isn't a geopoint property in the data dictionary
{
'key': 'not_coords',
'value': '-33.8561 151.2152 0 0'
},
)
def setUp(self):
self.domain = create_domain(DOMAIN)
self.user = CommCareUser.create(DOMAIN, USERNAME, PASSWORD, None, None)
initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO)
CaseSearchConfig.objects.get_or_create(pk=DOMAIN, enabled=True)
delete_all_cases()
self.case_id = uuid4().hex
_, [self.case] = post_case_blocks([CaseBlock(
create=True,
case_id=self.case_id,
case_type=CASE_TYPE,
case_name=CASE_NAME,
external_id=CASE_NAME,
user_id=OWNER_ID,
owner_id=OWNER_ID,
update={'opened_by': OWNER_ID},
).as_xml()], {'domain': DOMAIN})
self.additional_case_id = uuid4().hex
_, [self.additional_case] = post_case_blocks([CaseBlock.deprecated_init(
create=True,
case_id=self.additional_case_id,
case_type=CASE_TYPE,
case_name="Bilbo Baggins",
external_id="Bilbo Baggins",
user_id=OWNER_ID,
owner_id=OWNER_ID,
update={'opened_by': OWNER_ID},
).as_xml()], {'domain': DOMAIN})
self.case_ids = set([self.case_id, self.additional_case_id])
domains_needing_search_index.clear()
CaseSearchReindexerFactory(domain=DOMAIN).build().reindex()
es = get_es_new()
es.indices.refresh(CASE_SEARCH_INDEX)
self.client = Client()
self.client.login(username=USERNAME, password=PASSWORD)
self.url = reverse('claim_case', kwargs={'domain': DOMAIN})
self.synclog = SyncLogSQL.objects.bulk_create([
self.make_synclog(self.domain, 'u1', '2022-04-12')
])[0]
self.synclog.doc['case_ids_on_phone'] = [self.case_id]
with patch('casexml.apps.phone.change_publishers.publish_synclog_saved'):
self.synclog.save()
class CaseSearchPillowTest(TestCase):
domain = 'meereen'
def setUp(self):
super(CaseSearchPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_cases()
self.elasticsearch = get_es_new()
self.pillow = get_case_pillow(skip_ucr=True)
ensure_index_deleted(CASE_SEARCH_INDEX)
# Bootstrap ES
initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO)
def tearDown(self):
ensure_index_deleted(CASE_SEARCH_INDEX)
CaseSearchConfig.objects.all().delete()
super(CaseSearchPillowTest, self).tearDown()
def test_case_search_pillow(self):
consumer = get_test_kafka_consumer(topics.CASE)
kafka_seq = self._get_kafka_seq()
case = self._make_case(case_properties={'foo': 'bar'})
producer.send_change(topics.CASE,
doc_to_change(case.to_json()).metadata)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# enable case search for domain
with patch(
'corehq.pillows.case_search.domain_needs_search_index',
new=MagicMock(
return_value=True)) as fake_case_search_enabled_for_domain:
# send to elasticsearch
self.pillow.process_changes(since=kafka_seq, forever=False)
fake_case_search_enabled_for_domain.assert_called_with(self.domain)
self._assert_case_in_es(self.domain, case)
def test_case_search_reindex_by_domain(self):
"""
Tests reindexing for a particular domain only
"""
other_domain = "yunkai"
CaseSearchConfig.objects.get_or_create(pk=other_domain, enabled=True)
domains_needing_search_index.clear()
desired_case = self._make_case(domain=other_domain)
undesired_case = self._make_case(domain=self.domain) # noqa
with self.assertRaises(CaseSearchNotEnabledException):
CaseSearchReindexerFactory(domain=self.domain).build().reindex()
CaseSearchReindexerFactory(domain=other_domain).build().reindex()
self._assert_case_in_es(other_domain, desired_case)
def test_delete_case_search_cases(self):
"""
Tests that cases are correctly removed from the es index
"""
other_domain = "braavos"
self._bootstrap_cases_in_es_for_domain(self.domain)
case = self._bootstrap_cases_in_es_for_domain(other_domain)
with self.assertRaises(TypeError):
delete_case_search_cases(None)
with self.assertRaises(TypeError):
delete_case_search_cases({})
# delete cases from one domain
delete_case_search_cases(self.domain)
# make sure the other domain's cases are still there
self._assert_case_in_es(other_domain, case)
# delete other domains cases
delete_case_search_cases(other_domain)
# make sure nothing is left
self._assert_index_empty()
@override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True)
def test_sql_case_search_pillow(self):
consumer = get_test_kafka_consumer(topics.CASE_SQL)
# have to get the seq id before the change is processed
kafka_seq = self._get_kafka_seq()
case = self._make_case(case_properties={'something': 'something_else'})
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# enable case search for domain
with patch(
'corehq.pillows.case_search.domain_needs_search_index',
new=MagicMock(
return_value=True)) as fake_case_search_enabled_for_domain:
# send to elasticsearch
self.pillow.process_changes(since=kafka_seq, forever=False)
fake_case_search_enabled_for_domain.assert_called_with(self.domain)
self._assert_case_in_es(self.domain, case)
def _get_kafka_seq(self):
# KafkaChangeFeed listens for multiple topics (case, case-sql) in the case search pillow,
# so we need to provide a dict of seqs to kafka
return get_multi_topic_offset([topics.CASE, topics.CASE_SQL])
def _make_case(self, domain=None, case_properties=None):
# make a case
case_id = uuid.uuid4().hex
case_name = 'case-name-{}'.format(uuid.uuid4().hex)
if domain is None:
domain = self.domain
case = create_and_save_a_case(domain, case_id, case_name,
case_properties)
return case
def _assert_case_in_es(self, domain, case):
# confirm change made it to elasticserach
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
results = CaseSearchES().run()
self.assertEqual(1, results.total)
case_doc = results.hits[0]
self.assertEqual(domain, case_doc['domain'])
self.assertEqual(case.case_id, case_doc['_id'])
self.assertEqual(case.name, case_doc['name'])
# Confirm change contains case_properties
self.assertItemsEqual(list(case_doc['case_properties'][0]),
['key', 'value'])
for case_property in case_doc['case_properties']:
key = case_property['key']
try:
self.assertEqual(
SPECIAL_CASE_PROPERTIES_MAP[key].value_getter(
case.to_json()),
case_property['value'],
)
except KeyError:
self.assertEqual(case.get_case_property(key),
case_property['value'])
def _assert_index_empty(self):
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
results = CaseSearchES().run()
self.assertEqual(0, results.total)
def _bootstrap_cases_in_es_for_domain(self, domain):
case = self._make_case(domain)
with patch('corehq.pillows.case_search.domains_needing_search_index',
MagicMock(return_value=[domain])):
CaseSearchReindexerFactory(domain=domain).build().reindex()
return case
def reindex_case_search_for_domain(domain):
CaseSearchReindexerFactory(domain=domain).build().reindex()
def _bootstrap_cases_in_es_for_domain(self, domain):
with patch('corehq.pillows.case_search.domains_needing_search_index',
MagicMock(return_value=[domain])):
CaseSearchReindexerFactory(domain=domain).build().reindex()
def migrate_domain(self, domain):
self.stdout.write('Migrating {}...\n'.format(domain))
ECD_MIGRATED_DOMAINS.set(domain, True, NAMESPACE_DOMAIN)
domains_needing_search_index.clear()
CaseSearchReindexerFactory(domain=domain).build().reindex()
self.stdout.write('Done...\n\n'.format(domain))
class CaseSearchPillowTest(TestCase):
domain = 'meereen'
case_type = 'my_case_type'
def setUp(self):
super(CaseSearchPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_cases()
self.elasticsearch = get_es_new()
self.pillow = get_case_pillow(skip_ucr=True)
ensure_index_deleted(CASE_SEARCH_INDEX)
# Bootstrap ES
initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO)
def tearDown(self):
ensure_index_deleted(CASE_SEARCH_INDEX)
CaseSearchConfig.objects.all().delete()
super(CaseSearchPillowTest, self).tearDown()
def test_case_search_reindex_by_domain(self):
"""
Tests reindexing for a particular domain only
"""
other_domain = "yunkai"
CaseSearchConfig.objects.get_or_create(pk=other_domain, enabled=True)
domains_needing_search_index.clear()
desired_case = self._make_case(domain=other_domain)
undesired_case = self._make_case(domain=self.domain) # noqa
with self.assertRaises(CaseSearchNotEnabledException):
CaseSearchReindexerFactory(domain=self.domain).build().reindex()
CaseSearchReindexerFactory(domain=other_domain).build().reindex()
self._assert_case_in_es(other_domain, desired_case)
def test_delete_case_search_cases(self):
"""
Tests that cases are correctly removed from the es index
"""
other_domain = "braavos"
self._bootstrap_cases_in_es_for_domain(self.domain)
case = self._bootstrap_cases_in_es_for_domain(other_domain)
with self.assertRaises(TypeError):
delete_case_search_cases(None)
with self.assertRaises(TypeError):
delete_case_search_cases({})
# delete cases from one domain
delete_case_search_cases(self.domain)
# make sure the other domain's cases are still there
self._assert_case_in_es(other_domain, case)
# delete other domains cases
delete_case_search_cases(other_domain)
# make sure nothing is left
self._assert_index_empty()
def test_sql_case_search_pillow(self):
consumer = get_test_kafka_consumer(topics.CASE_SQL)
# have to get the seq id before the change is processed
kafka_seq = self._get_kafka_seq()
case = self._make_case(case_properties={'something': 'something_else'})
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# enable case search for domain
with patch(
'corehq.pillows.case_search.domain_needs_search_index',
new=MagicMock(
return_value=True)) as fake_case_search_enabled_for_domain:
# send to elasticsearch
self.pillow.process_changes(since=kafka_seq, forever=False)
fake_case_search_enabled_for_domain.assert_called_with(self.domain)
self._assert_case_in_es(self.domain, case)
def _get_kafka_seq(self):
return get_topic_offset(topics.CASE_SQL)
@flag_enabled('USH_CASE_CLAIM_UPDATES')
def test_geopoint_property(self):
CaseSearchConfig.objects.get_or_create(pk=self.domain, enabled=True)
domains_needing_search_index.clear()
self._make_data_dictionary(
gps_properties=['coords', 'short_coords', 'other_coords'])
case = self._make_case(
case_properties={
'coords': '-33.8561 151.2152 0 0',
'short_coords': '-33.8561 151.2152',
'other_coords': '42 Wallaby Way',
'not_coords': '-33.8561 151.2152 0 0',
})
CaseSearchReindexerFactory(domain=self.domain).build().reindex()
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
es_case = CaseSearchES().doc_id(case.case_id).run().hits[0]
self.assertEqual(
self._get_prop(es_case['case_properties'], 'coords'),
{
'key': 'coords',
'value': '-33.8561 151.2152 0 0',
'geopoint_value': {
'lat': -33.8561,
'lon': 151.2152
},
},
)
self.assertEqual(
self._get_prop(es_case['case_properties'], 'short_coords'),
{
'key': 'short_coords',
'value': '-33.8561 151.2152',
'geopoint_value': {
'lat': -33.8561,
'lon': 151.2152
},
},
)
self.assertEqual(
self._get_prop(es_case['case_properties'], 'other_coords'),
# The value here isn't a valid geopoint
{
'key': 'other_coords',
'value': '42 Wallaby Way',
'geopoint_value': None
},
)
self.assertEqual(
self._get_prop(es_case['case_properties'], 'not_coords'),
# This isn't a geopoint property in the data dictionary
{
'key': 'not_coords',
'value': '-33.8561 151.2152 0 0'
},
)
def _make_data_dictionary(self, gps_properties):
case_type = CaseType.objects.create(name=self.case_type,
domain=self.domain)
for prop in gps_properties:
CaseProperty.objects.create(case_type=case_type,
data_type="gps",
name=prop)
@staticmethod
def _get_prop(case_properties, key):
return [prop for prop in case_properties if prop['key'] == key][0]
def _make_case(self, domain=None, case_properties=None):
# make a case
case_id = uuid.uuid4().hex
case_name = 'case-name-{}'.format(uuid.uuid4().hex)
if domain is None:
domain = self.domain
case = create_and_save_a_case(domain, case_id, case_name,
case_properties, self.case_type)
return case
def _assert_case_in_es(self, domain, case):
# confirm change made it to elasticsearch
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
results = CaseSearchES().run()
self.assertEqual(1, results.total)
case_doc = results.hits[0]
self.assertEqual(domain, case_doc['domain'])
self.assertEqual(case.case_id, case_doc['_id'])
self.assertEqual(case.name, case_doc['name'])
# Confirm change contains case_properties
self.assertItemsEqual(list(case_doc['case_properties'][0]),
['key', 'value'])
for case_property in case_doc['case_properties']:
key = case_property['key']
try:
self.assertEqual(
SPECIAL_CASE_PROPERTIES_MAP[key].value_getter(
case.to_json()),
case_property['value'],
)
except KeyError:
self.assertEqual(case.get_case_property(key),
case_property['value'])
def _assert_index_empty(self):
self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
results = CaseSearchES().run()
self.assertEqual(0, results.total)
def _bootstrap_cases_in_es_for_domain(self, domain):
case = self._make_case(domain)
with patch('corehq.pillows.case_search.domains_needing_search_index',
MagicMock(return_value=[domain])):
CaseSearchReindexerFactory(domain=domain).build().reindex()
return case
