def test_case_search_reindex_by_domain(self): """ Tests reindexing for a particular domain only """ other_domain = "yunkai" CaseSearchConfig.objects.get_or_create(pk=other_domain, enabled=True) domains_needing_search_index.clear() desired_case = self._make_case(domain=other_domain) undesired_case = self._make_case(domain=self.domain) # noqa with self.assertRaises(CaseSearchNotEnabledException): CaseSearchReindexerFactory(domain=self.domain).build().reindex() CaseSearchReindexerFactory(domain=other_domain).build().reindex() self._assert_case_in_es(other_domain, desired_case)
def setUp(self): self.domain = create_domain(DOMAIN) self.user = CommCareUser.create(DOMAIN, USERNAME, PASSWORD, None, None) initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO) CaseSearchConfig.objects.get_or_create(pk=DOMAIN, enabled=True) delete_all_cases() self.case_id = uuid4().hex _, [self.case] = post_case_blocks([ CaseBlock.deprecated_init( create=True, case_id=self.case_id, case_type=CASE_TYPE, case_name=CASE_NAME, external_id=CASE_NAME, user_id=OWNER_ID, owner_id=OWNER_ID, update={ 'opened_by': OWNER_ID }, ).as_xml() ], {'domain': DOMAIN}) domains_needing_search_index.clear() CaseSearchReindexerFactory(domain=DOMAIN).build().reindex() es = get_es_new() es.indices.refresh(CASE_SEARCH_INDEX)
def _bootstrap_cases_in_es_for_domain(self, domain, input_cases): for case in input_cases: index = case.pop('index', None) self._make_case(domain, case, index=index) with patch('corehq.pillows.case_search.domains_needing_search_index', MagicMock(return_value=[domain])): CaseSearchReindexerFactory(domain=domain).build().reindex() self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX)
def test_geopoint_property(self): CaseSearchConfig.objects.get_or_create(pk=self.domain, enabled=True) domains_needing_search_index.clear() self._make_data_dictionary( gps_properties=['coords', 'short_coords', 'other_coords']) case = self._make_case( case_properties={ 'coords': '-33.8561 151.2152 0 0', 'short_coords': '-33.8561 151.2152', 'other_coords': '42 Wallaby Way', 'not_coords': '-33.8561 151.2152 0 0', }) CaseSearchReindexerFactory(domain=self.domain).build().reindex() self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) es_case = CaseSearchES().doc_id(case.case_id).run().hits[0] self.assertEqual( self._get_prop(es_case['case_properties'], 'coords'), { 'key': 'coords', 'value': '-33.8561 151.2152 0 0', 'geopoint_value': { 'lat': -33.8561, 'lon': 151.2152 }, }, ) self.assertEqual( self._get_prop(es_case['case_properties'], 'short_coords'), { 'key': 'short_coords', 'value': '-33.8561 151.2152', 'geopoint_value': { 'lat': -33.8561, 'lon': 151.2152 }, }, ) self.assertEqual( self._get_prop(es_case['case_properties'], 'other_coords'), # The value here isn't a valid geopoint { 'key': 'other_coords', 'value': '42 Wallaby Way', 'geopoint_value': None }, ) self.assertEqual( self._get_prop(es_case['case_properties'], 'not_coords'), # This isn't a geopoint property in the data dictionary { 'key': 'not_coords', 'value': '-33.8561 151.2152 0 0' }, )
def setUp(self): self.domain = create_domain(DOMAIN) self.user = CommCareUser.create(DOMAIN, USERNAME, PASSWORD, None, None) initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO) CaseSearchConfig.objects.get_or_create(pk=DOMAIN, enabled=True) delete_all_cases() self.case_id = uuid4().hex _, [self.case] = post_case_blocks([CaseBlock( create=True, case_id=self.case_id, case_type=CASE_TYPE, case_name=CASE_NAME, external_id=CASE_NAME, user_id=OWNER_ID, owner_id=OWNER_ID, update={'opened_by': OWNER_ID}, ).as_xml()], {'domain': DOMAIN}) self.additional_case_id = uuid4().hex _, [self.additional_case] = post_case_blocks([CaseBlock.deprecated_init( create=True, case_id=self.additional_case_id, case_type=CASE_TYPE, case_name="Bilbo Baggins", external_id="Bilbo Baggins", user_id=OWNER_ID, owner_id=OWNER_ID, update={'opened_by': OWNER_ID}, ).as_xml()], {'domain': DOMAIN}) self.case_ids = set([self.case_id, self.additional_case_id]) domains_needing_search_index.clear() CaseSearchReindexerFactory(domain=DOMAIN).build().reindex() es = get_es_new() es.indices.refresh(CASE_SEARCH_INDEX) self.client = Client() self.client.login(username=USERNAME, password=PASSWORD) self.url = reverse('claim_case', kwargs={'domain': DOMAIN}) self.synclog = SyncLogSQL.objects.bulk_create([ self.make_synclog(self.domain, 'u1', '2022-04-12') ])[0] self.synclog.doc['case_ids_on_phone'] = [self.case_id] with patch('casexml.apps.phone.change_publishers.publish_synclog_saved'): self.synclog.save()
class CaseSearchPillowTest(TestCase): domain = 'meereen' def setUp(self): super(CaseSearchPillowTest, self).setUp() FormProcessorTestUtils.delete_all_cases() self.elasticsearch = get_es_new() self.pillow = get_case_pillow(skip_ucr=True) ensure_index_deleted(CASE_SEARCH_INDEX) # Bootstrap ES initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO) def tearDown(self): ensure_index_deleted(CASE_SEARCH_INDEX) CaseSearchConfig.objects.all().delete() super(CaseSearchPillowTest, self).tearDown() def test_case_search_pillow(self): consumer = get_test_kafka_consumer(topics.CASE) kafka_seq = self._get_kafka_seq() case = self._make_case(case_properties={'foo': 'bar'}) producer.send_change(topics.CASE, doc_to_change(case.to_json()).metadata) # confirm change made it to kafka message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # enable case search for domain with patch( 'corehq.pillows.case_search.domain_needs_search_index', new=MagicMock( return_value=True)) as fake_case_search_enabled_for_domain: # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) fake_case_search_enabled_for_domain.assert_called_with(self.domain) self._assert_case_in_es(self.domain, case) def test_case_search_reindex_by_domain(self): """ Tests reindexing for a particular domain only """ other_domain = "yunkai" CaseSearchConfig.objects.get_or_create(pk=other_domain, enabled=True) domains_needing_search_index.clear() desired_case = self._make_case(domain=other_domain) undesired_case = self._make_case(domain=self.domain) # noqa with self.assertRaises(CaseSearchNotEnabledException): CaseSearchReindexerFactory(domain=self.domain).build().reindex() CaseSearchReindexerFactory(domain=other_domain).build().reindex() self._assert_case_in_es(other_domain, desired_case) def test_delete_case_search_cases(self): """ Tests that cases are correctly removed from the es index """ other_domain = "braavos" self._bootstrap_cases_in_es_for_domain(self.domain) case = self._bootstrap_cases_in_es_for_domain(other_domain) with self.assertRaises(TypeError): delete_case_search_cases(None) with self.assertRaises(TypeError): delete_case_search_cases({}) # delete cases from one domain delete_case_search_cases(self.domain) # make sure the other domain's cases are still there self._assert_case_in_es(other_domain, case) # delete other domains cases delete_case_search_cases(other_domain) # make sure nothing is left self._assert_index_empty() @override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True) def test_sql_case_search_pillow(self): consumer = get_test_kafka_consumer(topics.CASE_SQL) # have to get the seq id before the change is processed kafka_seq = self._get_kafka_seq() case = self._make_case(case_properties={'something': 'something_else'}) # confirm change made it to kafka message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # enable case search for domain with patch( 'corehq.pillows.case_search.domain_needs_search_index', new=MagicMock( return_value=True)) as fake_case_search_enabled_for_domain: # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) fake_case_search_enabled_for_domain.assert_called_with(self.domain) self._assert_case_in_es(self.domain, case) def _get_kafka_seq(self): # KafkaChangeFeed listens for multiple topics (case, case-sql) in the case search pillow, # so we need to provide a dict of seqs to kafka return get_multi_topic_offset([topics.CASE, topics.CASE_SQL]) def _make_case(self, domain=None, case_properties=None): # make a case case_id = uuid.uuid4().hex case_name = 'case-name-{}'.format(uuid.uuid4().hex) if domain is None: domain = self.domain case = create_and_save_a_case(domain, case_id, case_name, case_properties) return case def _assert_case_in_es(self, domain, case): # confirm change made it to elasticserach self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) results = CaseSearchES().run() self.assertEqual(1, results.total) case_doc = results.hits[0] self.assertEqual(domain, case_doc['domain']) self.assertEqual(case.case_id, case_doc['_id']) self.assertEqual(case.name, case_doc['name']) # Confirm change contains case_properties self.assertItemsEqual(list(case_doc['case_properties'][0]), ['key', 'value']) for case_property in case_doc['case_properties']: key = case_property['key'] try: self.assertEqual( SPECIAL_CASE_PROPERTIES_MAP[key].value_getter( case.to_json()), case_property['value'], ) except KeyError: self.assertEqual(case.get_case_property(key), case_property['value']) def _assert_index_empty(self): self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) results = CaseSearchES().run() self.assertEqual(0, results.total) def _bootstrap_cases_in_es_for_domain(self, domain): case = self._make_case(domain) with patch('corehq.pillows.case_search.domains_needing_search_index', MagicMock(return_value=[domain])): CaseSearchReindexerFactory(domain=domain).build().reindex() return case
def reindex_case_search_for_domain(domain): CaseSearchReindexerFactory(domain=domain).build().reindex()
def _bootstrap_cases_in_es_for_domain(self, domain): with patch('corehq.pillows.case_search.domains_needing_search_index', MagicMock(return_value=[domain])): CaseSearchReindexerFactory(domain=domain).build().reindex()
def migrate_domain(self, domain): self.stdout.write('Migrating {}...\n'.format(domain)) ECD_MIGRATED_DOMAINS.set(domain, True, NAMESPACE_DOMAIN) domains_needing_search_index.clear() CaseSearchReindexerFactory(domain=domain).build().reindex() self.stdout.write('Done...\n\n'.format(domain))
class CaseSearchPillowTest(TestCase): domain = 'meereen' case_type = 'my_case_type' def setUp(self): super(CaseSearchPillowTest, self).setUp() FormProcessorTestUtils.delete_all_cases() self.elasticsearch = get_es_new() self.pillow = get_case_pillow(skip_ucr=True) ensure_index_deleted(CASE_SEARCH_INDEX) # Bootstrap ES initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO) def tearDown(self): ensure_index_deleted(CASE_SEARCH_INDEX) CaseSearchConfig.objects.all().delete() super(CaseSearchPillowTest, self).tearDown() def test_case_search_reindex_by_domain(self): """ Tests reindexing for a particular domain only """ other_domain = "yunkai" CaseSearchConfig.objects.get_or_create(pk=other_domain, enabled=True) domains_needing_search_index.clear() desired_case = self._make_case(domain=other_domain) undesired_case = self._make_case(domain=self.domain) # noqa with self.assertRaises(CaseSearchNotEnabledException): CaseSearchReindexerFactory(domain=self.domain).build().reindex() CaseSearchReindexerFactory(domain=other_domain).build().reindex() self._assert_case_in_es(other_domain, desired_case) def test_delete_case_search_cases(self): """ Tests that cases are correctly removed from the es index """ other_domain = "braavos" self._bootstrap_cases_in_es_for_domain(self.domain) case = self._bootstrap_cases_in_es_for_domain(other_domain) with self.assertRaises(TypeError): delete_case_search_cases(None) with self.assertRaises(TypeError): delete_case_search_cases({}) # delete cases from one domain delete_case_search_cases(self.domain) # make sure the other domain's cases are still there self._assert_case_in_es(other_domain, case) # delete other domains cases delete_case_search_cases(other_domain) # make sure nothing is left self._assert_index_empty() def test_sql_case_search_pillow(self): consumer = get_test_kafka_consumer(topics.CASE_SQL) # have to get the seq id before the change is processed kafka_seq = self._get_kafka_seq() case = self._make_case(case_properties={'something': 'something_else'}) # confirm change made it to kafka message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # enable case search for domain with patch( 'corehq.pillows.case_search.domain_needs_search_index', new=MagicMock( return_value=True)) as fake_case_search_enabled_for_domain: # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) fake_case_search_enabled_for_domain.assert_called_with(self.domain) self._assert_case_in_es(self.domain, case) def _get_kafka_seq(self): return get_topic_offset(topics.CASE_SQL) @flag_enabled('USH_CASE_CLAIM_UPDATES') def test_geopoint_property(self): CaseSearchConfig.objects.get_or_create(pk=self.domain, enabled=True) domains_needing_search_index.clear() self._make_data_dictionary( gps_properties=['coords', 'short_coords', 'other_coords']) case = self._make_case( case_properties={ 'coords': '-33.8561 151.2152 0 0', 'short_coords': '-33.8561 151.2152', 'other_coords': '42 Wallaby Way', 'not_coords': '-33.8561 151.2152 0 0', }) CaseSearchReindexerFactory(domain=self.domain).build().reindex() self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) es_case = CaseSearchES().doc_id(case.case_id).run().hits[0] self.assertEqual( self._get_prop(es_case['case_properties'], 'coords'), { 'key': 'coords', 'value': '-33.8561 151.2152 0 0', 'geopoint_value': { 'lat': -33.8561, 'lon': 151.2152 }, }, ) self.assertEqual( self._get_prop(es_case['case_properties'], 'short_coords'), { 'key': 'short_coords', 'value': '-33.8561 151.2152', 'geopoint_value': { 'lat': -33.8561, 'lon': 151.2152 }, }, ) self.assertEqual( self._get_prop(es_case['case_properties'], 'other_coords'), # The value here isn't a valid geopoint { 'key': 'other_coords', 'value': '42 Wallaby Way', 'geopoint_value': None }, ) self.assertEqual( self._get_prop(es_case['case_properties'], 'not_coords'), # This isn't a geopoint property in the data dictionary { 'key': 'not_coords', 'value': '-33.8561 151.2152 0 0' }, ) def _make_data_dictionary(self, gps_properties): case_type = CaseType.objects.create(name=self.case_type, domain=self.domain) for prop in gps_properties: CaseProperty.objects.create(case_type=case_type, data_type="gps", name=prop) @staticmethod def _get_prop(case_properties, key): return [prop for prop in case_properties if prop['key'] == key][0] def _make_case(self, domain=None, case_properties=None): # make a case case_id = uuid.uuid4().hex case_name = 'case-name-{}'.format(uuid.uuid4().hex) if domain is None: domain = self.domain case = create_and_save_a_case(domain, case_id, case_name, case_properties, self.case_type) return case def _assert_case_in_es(self, domain, case): # confirm change made it to elasticsearch self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) results = CaseSearchES().run() self.assertEqual(1, results.total) case_doc = results.hits[0] self.assertEqual(domain, case_doc['domain']) self.assertEqual(case.case_id, case_doc['_id']) self.assertEqual(case.name, case_doc['name']) # Confirm change contains case_properties self.assertItemsEqual(list(case_doc['case_properties'][0]), ['key', 'value']) for case_property in case_doc['case_properties']: key = case_property['key'] try: self.assertEqual( SPECIAL_CASE_PROPERTIES_MAP[key].value_getter( case.to_json()), case_property['value'], ) except KeyError: self.assertEqual(case.get_case_property(key), case_property['value']) def _assert_index_empty(self): self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) results = CaseSearchES().run() self.assertEqual(0, results.total) def _bootstrap_cases_in_es_for_domain(self, domain): case = self._make_case(domain) with patch('corehq.pillows.case_search.domains_needing_search_index', MagicMock(return_value=[domain])): CaseSearchReindexerFactory(domain=domain).build().reindex() return case