def post(self, request, *args, **kwargs): from corehq.apps.es.case_search import CaseSearchES if not case_search_enabled_for_domain(self.domain): raise BadRequest("Domain does not have case search enabled") query = json.loads(request.POST.get('q')) case_type = query.get('type') owner_id = query.get('owner_id') search_params = query.get('parameters', []) query_addition = query.get("customQueryAddition", None) search = CaseSearchES() search = search.domain(self.domain).is_closed(False) if case_type: search = search.case_type(case_type) if owner_id: search = search.owner(owner_id) for param in search_params: value = re.sub(param.get('regex', ''), '', param.get('value')) search = search.case_property_query( param.get('key'), value, clause=param.get('clause'), fuzzy=param.get('fuzzy'), ) if query_addition: addition = CaseSearchQueryAddition.objects.get(id=query_addition, domain=self.domain) new_query = merge_queries(search.get_query(), addition.query_addition) search = search.set_query(new_query) search_results = search.values() return json_response({'values': search_results})
def _parent_property_lookup(node): """given a node of the form `parent/foo = 'thing'`, return all case_ids where `foo = thing` """ if isinstance(node.right, Step): _raise_step_RHS(node) new_query = "{} {} '{}'".format(serialize(node.left.right), node.op, node.right) es_query = CaseSearchES().domain(domain).xpath_query(domain, new_query) if es_query.count() > MAX_RELATED_CASES: raise CaseFilterError( _("The related case lookup you are trying to perform would return too many cases" ), new_query) return es_query.scroll_ids()
def _parent_property_lookup(node): """given a node of the form `parent/foo = 'thing'`, return all case_ids where `foo = thing` """ if isinstance(node.right, Step): _raise_step_RHS(node) new_query = "{} {} '{}'".format(serialize(node.left.right), node.op, node.right) es_query = CaseSearchES().domain(domain).xpath_query(domain, new_query) if es_query.count() > MAX_RELATED_CASES: raise CaseFilterError( _("The related case lookup you are trying to perform would return too many cases"), new_query ) return es_query.scroll_ids()
def _parent_property_lookup(node): """given a node of the form `parent/foo = 'thing'`, return all case_ids where `foo = thing` """ es_filter = _comparison_raw(node.left.right, node.op, node.right, node) es_query = CaseSearchES().domain(context.domain).filter(es_filter) if es_query.count() > MAX_RELATED_CASES: new_query = '{} {} "{}"'.format(serialize(node.left.right), node.op, node.right) raise TooManyRelatedCasesError( _("The related case lookup you are trying to perform would return too many cases" ), new_query) return es_query.scroll_ids()
def post(self, request, *args, **kwargs): from corehq.apps.es.case_search import CaseSearchES if not case_search_enabled_for_domain(self.domain): raise BadRequest("Domain does not have case search enabled") query = json.loads(request.POST.get('q')) case_type = query.get('type') owner_id = query.get('owner_id') search_params = query.get('parameters', []) xpath = query.get("xpath") search = CaseSearchES() search = search.domain(self.domain).size(10) if case_type: search = search.case_type(case_type) if owner_id: search = search.owner(owner_id) for param in search_params: value = re.sub(param.get('regex', ''), '', param.get('value')) if '/' in param.get('key'): query = '{} = "{}"'.format(param.get('key'), value) search = search.xpath_query(self.domain, query, fuzzy=param.get('fuzzy')) else: search = search.case_property_query( param.get('key'), value, clause=param.get('clause'), fuzzy=param.get('fuzzy'), ) if xpath: search = search.xpath_query(self.domain, xpath) include_profile = request.POST.get("include_profile", False) if include_profile: search = search.enable_profiling() search_results = search.run() return json_response({ 'values': search_results.raw_hits, 'count': search_results.total, 'took': search_results.raw['took'], 'query': search_results.query.dumps(pretty=True), 'profile': json.dumps(search_results.raw.get('profile', {}), indent=2), })
def test_numeric_range_query(self): self._assert_query_runs_correctly( self.domain, [ { '_id': 'c1', 'num': '1' }, { '_id': 'c2', 'num': '2' }, { '_id': 'c3', 'num': '3' }, { '_id': 'c4', 'num': '4' }, ], CaseSearchES().domain( self.domain).numeric_range_case_property_query('num', gte=2, lte=3), 'num <= 3 and num >= 2', ['c2', 'c3'])
def _parent_property_lookup(node): """given a node of the form `parent/foo = 'thing'`, return all case_ids where `foo = thing` """ if isinstance(node.right, Step): _raise_step_RHS(node) new_query = "{} {} '{}'".format(serialize(node.left.right), node.op, node.right) return CaseSearchES().domain(domain).xpath_query(domain, new_query).scroll_ids()
def _get_initial_search_es(self): search_es = (CaseSearchES() .domain(self.domain) .case_type(self.case_type) .is_closed(False) .size(CASE_SEARCH_MAX_RESULTS)) return search_es
def get_related_case_results(domain, cases, paths): """ Given a set of cases and a set of case property paths, fetches ES documents for all cases referenced by those paths. """ if not cases: return [] results_cache = {} for path in paths: current_cases = cases parts = path.split("/") for index, identifier in enumerate(parts): fragment = "/".join(parts[:index + 1]) if fragment in results_cache: current_cases = results_cache[fragment] else: indices = [ case.get_index(identifier) for case in current_cases ] related_case_ids = {i.referenced_id for i in indices if i} results = CaseSearchES().domain(domain).case_ids( related_case_ids).run().hits current_cases = [ CommCareCase.wrap(flatten_result(result)) for result in results ] results_cache[fragment] = current_cases
def post(self, request, *args, **kwargs): from corehq.apps.es.case_search import CaseSearchES if not case_search_enabled_for_domain(self.domain): raise BadRequest("Domain does not have case search enabled") query = json.loads(request.POST.get('q')) case_type = query.get('type') search_params = query.get('parameters', []) search = CaseSearchES() search = search.domain(self.domain).is_closed(False) if case_type: search = search.case_type(case_type) for param in search_params: search = search.case_property_query(**param) search_results = search.values() return json_response({'values': search_results})
def test_get_related_case_results(self): # Note that cases must be defined before other cases can reference them cases = [ { '_id': 'c1', 'case_type': 'monster', 'description': 'grandparent of first person' }, { '_id': 'c2', 'case_type': 'monster', 'description': 'parent of first person', 'index': { 'parent': ('monster', 'c1') } }, { '_id': 'c3', 'case_type': 'monster', 'description': 'parent of host' }, { '_id': 'c4', 'case_type': 'monster', 'description': 'host of second person', 'index': { 'parent': ('monster', 'c3') } }, { '_id': 'c5', 'description': 'first person', 'index': { 'parent': ('monster', 'c2') } }, { '_id': 'c6', 'description': 'second person', 'index': { 'host': ('monster', 'c4') } }, ] self._bootstrap_cases_in_es_for_domain(self.domain, cases) hits = CaseSearchES().domain(self.domain).case_type( self.case_type).run().hits cases = [CommCareCase.wrap(flatten_result(result)) for result in hits] self.assertEqual({case.case_id for case in cases}, {'c5', 'c6'}) self._assert_related_case_ids(cases, set(), set()) self._assert_related_case_ids(cases, {"parent"}, {"c2"}) self._assert_related_case_ids(cases, {"host"}, {"c4"}) self._assert_related_case_ids(cases, {"parent/parent"}, {"c1"}) self._assert_related_case_ids(cases, {"host/parent"}, {"c3"}) self._assert_related_case_ids(cases, {"host", "parent"}, {"c2", "c4"}) self._assert_related_case_ids(cases, {"host", "parent/parent"}, {"c4", "c1"})
def search(request, domain): """ Accepts search criteria as GET params, e.g. "https://www.commcarehq.org/a/domain/phone/search/?a=b&c=d" Returns results as a fixture with the same structure as a casedb instance. """ criteria = request.GET.dict() try: case_type = criteria.pop('case_type') except KeyError: return HttpResponse('Search request must specify case type', status=400) try: include_closed = criteria.pop('include_closed') except KeyError: include_closed = False search_es = (CaseSearchES().domain(domain).case_type(case_type).size( CASE_SEARCH_MAX_RESULTS)) if include_closed != 'True': search_es = search_es.is_closed(False) try: config = CaseSearchConfig.objects.get(domain=domain) except CaseSearchConfig.DoesNotExist as e: from corehq.util.soft_assert import soft_assert _soft_assert = soft_assert(to="{}@{}.com".format('frener', 'dimagi'), notify_admins=False, send_to_ops=False) _soft_assert( False, u"Someone in domain: {} tried accessing case search without a config" .format(domain), e) config = CaseSearchConfig(domain=domain) query_addition_id = criteria.pop(SEARCH_QUERY_ADDITION_KEY, None) fuzzies = config.config.get_fuzzy_properties_for_case_type(case_type) for key, value in criteria.items(): search_es = search_es.case_property_query(key, value, fuzzy=(key in fuzzies)) query_addition_debug_details = {} try: search_es = _add_case_search_addition(request, domain, search_es, query_addition_id, query_addition_debug_details) except QueryMergeException as e: return _handle_query_merge_exception(request, e) try: results = search_es.values() except Exception as e: return _handle_es_exception(request, e, query_addition_debug_details) # Even if it's a SQL domain, we just need to render the results as cases, so CommCareCase.wrap will be fine cases = [CommCareCase.wrap(flatten_result(result)) for result in results] fixtures = CaseDBFixture(cases).fixture return HttpResponse(fixtures, content_type="text/xml")
def _get_assigned_cases(checkin_case): query = (CaseSearchES().domain(checkin_case.domain).filter( filters.OR(case_type("patient"), case_type("contact"))).case_property_query( "assigned_to_primary_checkin_case_id", checkin_case.case_id)) return [CommCareCase.wrap(flatten_result(hit)) for hit in query.run().hits]
def delete_case_search_cases(domain): if domain is None or isinstance(domain, dict): raise TypeError("Domain attribute is required") case_search = ElasticCaseSearch() ElasticManageAdapter().index_refresh(case_search.index_name) case_ids = CaseSearchES().domain(domain).values_list('_id', flat=True) case_search.bulk_delete(case_ids)
def _assert_query_runs_correctly(self, domain, input_cases, query, xpath_query, output): self._bootstrap_cases_in_es_for_domain(domain, input_cases) self.assertItemsEqual(query.get_ids(), output) if xpath_query: self.assertItemsEqual( CaseSearchES().xpath_query(self.domain, xpath_query).get_ids(), output)
def _get_previous_explosions(self): results = CaseSearchES().domain(self.domain).aggregation( CasePropertyAggregation('explosions', 'cc_explosion_id')).size(0).run() return sorted( list(results.aggregations.explosions.counts_by_bucket().items()), key=lambda x: -x[1] # sorted by number of cases )
def get_case_ids(self, case_type): """ All open and closed person cases with person.dataset = 'real' and person.enrolled_in_private != 'true' """ return (CaseSearchES(es_instance_alias=ES_EXPORT_INSTANCE).domain( DOMAIN).case_type(case_type).case_property_query( ENROLLED_IN_PRIVATE, 'true', clause=queries.MUST_NOT).case_property_query( "dataset", 'real').get_ids()[0:10])
def post(self, request, *args, **kwargs): from corehq.apps.es.case_search import CaseSearchES if not case_search_enabled_for_domain(self.domain): raise BadRequest("Domain does not have case search enabled") query = json.loads(request.POST.get('q')) case_type = query.get('type') owner_id = query.get('owner_id') search_params = query.get('parameters', []) query_addition = query.get("customQueryAddition", None) include_closed = query.get("includeClosed", False) xpath = query.get("xpath") search = CaseSearchES() search = search.domain(self.domain).size(10) if not include_closed: search = search.is_closed(False) if case_type: search = search.case_type(case_type) if owner_id: search = search.owner(owner_id) for param in search_params: value = re.sub(param.get('regex', ''), '', param.get('value')) search = search.case_property_query( param.get('key'), value, clause=param.get('clause'), fuzzy=param.get('fuzzy'), ) if query_addition: addition = CaseSearchQueryAddition.objects.get(id=query_addition, domain=self.domain) new_query = merge_queries(search.get_query(), addition.query_addition) search = search.set_query(new_query) if xpath: search = search.xpath_query(self.domain, xpath) search_results = search.run() return json_response({ 'values': search_results.raw_hits, 'count': search_results.total, 'took': search_results.raw['took'], 'query': search_results.query.dumps(pretty=True), })
def test_full_text_query(self): self._assert_query_runs_correctly( self.domain, [ {'_id': 'c1', 'description': 'redbeards are red'}, {'_id': 'c2', 'description': 'blackbeards are black'}, ], CaseSearchES().domain(self.domain).filter(case_property_text_query('description', 'red')), None, ['c1'] )
def test_blacklisted_owner_ids(self): self._assert_query_runs_correctly( self.domain, [ {'_id': 'c1', 'owner_id': '123'}, {'_id': 'c2', 'owner_id': '234'}, ], CaseSearchES().domain(self.domain).blacklist_owner_id('123'), None, ['c2'] )
def get_usercase_from_checkin(checkin_case): username = checkin_case.get_case_property("username") query = (CaseSearchES().domain( checkin_case.domain).case_type(USERCASE_TYPE).case_property_query( "username", username)) results = query.run().hits if not results: return None return CommCareCase.wrap(flatten_result(results[0]))
def test_simple_case_property_query(self): self._assert_query_runs_correctly( self.domain, [ {'_id': 'c1', 'foo': 'redbeard'}, {'_id': 'c2', 'foo': 'blackbeard'}, ], CaseSearchES().domain(self.domain).case_property_query("foo", "redbeard"), "foo = 'redbeard'", ['c1'] )
def _assert_query_runs_correctly(self, domain, input_cases, query, xpath_query, output): for case in input_cases: self._make_case(domain, case) self._bootstrap_cases_in_es_for_domain(domain) self.elasticsearch.indices.refresh(CASE_SEARCH_INDEX) self.assertItemsEqual(query.get_ids(), output) if xpath_query: self.assertItemsEqual( CaseSearchES().xpath_query(self.domain, xpath_query).get_ids(), output)
def find_case_ids(self, domain): query = (CaseSearchES().domain(domain).filter( exact_case_property_text_query('current_status', 'closed')).filter( case_property_missing("all_activity_complete_date")).case_type( self.case_type).owner(INACTIVE_LOCATION_IDS)) if self.case_type == 'contact': query = (query.modified_range(gte=datetime.date(2022, 2, 2)).NOT( exact_case_property_text_query('final_disposition', 'converted_to_pui'))) return list(query.scroll_ids())
def get_case_ids(self, case_type): """ All open and closed episode cases whose host/host = a person case (open or closed) with person.dataset = 'real' and person.enrolled_in_private != 'true' """ return (CaseSearchES().domain(DOMAIN).case_type( case_type).case_property_query( ENROLLED_IN_PRIVATE, 'true', clause=queries.MUST_NOT).case_property_query( "episode_type", DSTB_EPISODE_TYPE, clause=queries.MUST).get_ids())
def test_missing_case_property(self): self._assert_query_runs_correctly( self.domain, [ {'_id': 'c2', 'foo': 'blackbeard'}, {'_id': 'c3', 'foo': ''}, {'_id': 'c4'}, ], CaseSearchES().domain(self.domain).filter(case_property_missing('foo')), "foo = ''", ['c3', 'c4'] )
def __init__(self, *args, **kwargs): super(BaseDataDump, self).__init__(*args, **kwargs) self.log_progress = None self.result_file_name = None self.case_type = None self.report = {} self.notes = {"Column Name": "Notes"} self.column_statuses = {"Column Name": "Column Status"} self.result_file_headers = ["Column Name"] self.recipient = None self.full = False self.case_search_instance = CaseSearchES(es_instance_alias=ES_EXPORT_INSTANCE).domain(DOMAIN)
def get_usercase_from_checkin(checkin_case): username = checkin_case.get_case_property("username") if not username: return None query = (CaseSearchES().domain( checkin_case.domain).case_type(USERCASE_TYPE).case_property_query( "username", username)) results = query.run().hits if not results: return None return wrap_case_search_hit(results[0])
def test_date_range_query(self): self._assert_query_runs_correctly( self.domain, [ {'_id': 'c1', 'dob': date(2020, 3, 1)}, {'_id': 'c2', 'dob': date(2020, 3, 2)}, {'_id': 'c3', 'dob': date(2020, 3, 3)}, {'_id': 'c4', 'dob': date(2020, 3, 4)}, ], CaseSearchES().domain(self.domain).date_range_case_property_query('dob', gte='2020-03-02', lte='2020-03-03'), "dob >= '2020-03-02' and dob <= '2020-03-03'", ['c2', 'c3'] )
def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) self.person_enikshay_ids = [] self.person_ids = [] self.episode_ids = [] self.max_attempts_in_sheet = 0 self.search = CaseSearchES().domain('enikshay').size( CASE_SEARCH_MAX_RESULTS) self.wb = Workbook() self.ws = self.wb.active self.ws.append([ 'Enikshay ID', 'Episode ID', 'Should be Notified', 'Nikshay Registered', 'Nikshay Private Registered', 'Nikshay ID', 'External ID', 'Nikshay ID history', 'Nikshay Error', 'Private Nikshay Error', # Forwarding ID 1, Attempts 1, Forwarding ID 2, Attempts 2 .. ])
def _get_assigned_cases(checkin_case): """ An assigned case is a case for which all of the following are true Case type patient or contact Exists in the same domain as the user case The case property assigned_to_primary_checkin_case_id equals an associated checkin case's case_id """ query = (CaseSearchES().domain(checkin_case.domain).filter( filters.OR(case_type("patient"), case_type("contact"))).case_property_query( "assigned_to_primary_checkin_case_id", checkin_case.case_id)) return [CommCareCase.wrap(flatten_result(hit)) for hit in query.run().hits]
def test_multiple_case_search_queries(self): query = (CaseSearchES().domain(self.domain) .case_property_query("foo", "redbeard") .case_property_query("parrot_name", "polly")) self._assert_query_runs_correctly( self.domain, [ {'_id': 'c1', 'foo': 'redbeard', 'parrot_name': 'polly'}, {'_id': 'c2', 'foo': 'blackbeard', 'parrot_name': 'polly'}, {'_id': 'c3', 'foo': 'redbeard', 'parrot_name': 'molly'} ], query, "foo = 'redbeard' and parrot_name = 'polly'", ['c1'] )
def test_multiple_case_search_queries_should_clause(self): query = (CaseSearchES().domain(self.domain) .case_property_query("foo", "redbeard") .case_property_query("parrot_name", "polly", clause="should")) self._assert_query_runs_correctly( self.domain, [ {'_id': 'c1', 'foo': 'redbeard', 'parrot_name': 'polly'}, {'_id': 'c2', 'foo': 'blackbeard', 'parrot_name': 'polly'}, {'_id': 'c3', 'foo': 'redbeard', 'parrot_name': 'molly'} ], query, None, ['c1', 'c3'] )
class TestCaseSearchES(ElasticTestMixin, SimpleTestCase): def setUp(self): self.es = CaseSearchES() def test_simple_case_property_query(self): json_output = { "query": { "filtered": { "filter": { "and": [ { "term": { "domain.exact": "swashbucklers" } }, { "match_all": {} } ] }, "query": { "bool": { "must": [ { "nested": { "path": "case_properties", "query": { "filtered": { "query": { "match_all": { } }, "filter": { "and": ( { "term": { "case_properties.key.exact": "name" } }, { "term": { "case_properties.value.exact": "redbeard" } } ) } } } } } ] } } } }, "size": SIZE_LIMIT } query = self.es.domain('swashbucklers').case_property_query("name", "redbeard") self.checkQuery(query, json_output) def test_multiple_case_search_queries(self): json_output = { "query": { "filtered": { "filter": { "and": [ { "term": { "domain.exact": "swashbucklers" } }, { "match_all": {} } ] }, "query": { "bool": { "must": [ { "nested": { "path": "case_properties", "query": { "filtered": { "filter": { "and": ( { "term": { "case_properties.key.exact": "name" } }, { "term": { "case_properties.value.exact": "redbeard" } } ) }, "query": { "match_all": { } } } } } } ], "should": [ { "nested": { "path": "case_properties", "query": { "filtered": { "filter": { "term": { "case_properties.key.exact": "parrot_name" } }, "query": { "match": { "case_properties.value": { "query": "polly", "fuzziness": "AUTO" } } } } } } }, { "nested": { "path": "case_properties", "query": { "filtered": { "filter": { "term": { "case_properties.key.exact": "parrot_name" } }, "query": { "match": { "case_properties.value": { "query": "polly", "fuzziness": "0" } } } } } } } ] } } } }, "size": SIZE_LIMIT } query = (self.es.domain('swashbucklers') .case_property_query("name", "redbeard") .case_property_query("parrot_name", "polly", clause="should", fuzzy=True)) self.checkQuery(query, json_output) def test_flatten_result(self): expected = {'name': 'blah', 'foo': 'bar', 'baz': 'buzz', RELEVANCE_SCORE: "1.095"} self.assertEqual( flatten_result( { "_score": "1.095", "_source": { 'name': 'blah', 'case_properties': [ {'key': '@case_id', 'value': 'should be removed'}, {'key': 'name', 'value': 'should be removed'}, {'key': 'case_name', 'value': 'should be removed'}, {'key': 'last_modified', 'value': 'should be removed'}, {'key': 'foo', 'value': 'bar'}, {'key': 'baz', 'value': 'buzz'}] } }, include_score=True ), expected ) def test_blacklisted_owner_ids(self): query = self.es.domain('swashbucklers').blacklist_owner_id('123').owner('234') expected = {'query': {'filtered': {'filter': {'and': [ {'term': {'domain.exact': 'swashbucklers'}}, {'not': {'term': {'owner_id': '123'}}}, {'term': {'owner_id': '234'}}, {'match_all': {}} ]}, "query": { "match_all": {} }}}, 'size': SIZE_LIMIT} self.checkQuery(query, expected)
def setUp(self): self.es = CaseSearchES()
class TestCaseSearchES(ElasticTestMixin, TestCase): def setUp(self): self.es = CaseSearchES() def test_simple_case_property_query(self): json_output = { "query": { "filtered": { "filter": { "and": [ { "term": { "domain.exact": "swashbucklers" } }, { "match_all": {} } ] }, "query": { "bool": { "must": [ { "nested": { "path": "case_properties", "query": { "filtered": { "filter": { "term": { "case_properties.key": "name" } }, "query": { "match": { "case_properties.value": { "query": "redbeard", "fuzziness": "0" } } } } } } } ] } } } }, "size": SIZE_LIMIT } query = self.es.domain('swashbucklers').case_property_query("name", "redbeard") self.checkQuery(query, json_output) def test_multiple_case_search_queries(self): json_output = { "query": { "filtered": { "filter": { "and": [ { "term": { "domain.exact": "swashbucklers" } }, { "match_all": {} } ] }, "query": { "bool": { "must": [ { "nested": { "path": "case_properties", "query": { "filtered": { "filter": { "term": { "case_properties.key": "name" } }, "query": { "match": { "case_properties.value": { "query": "redbeard", "fuzziness": "0" } } } } } } } ], "should": [ { "nested": { "path": "case_properties", "query": { "filtered": { "filter": { "term": { "case_properties.key": "parrot_name" } }, "query": { "match": { "case_properties.value": { "query": "polly", "fuzziness": "AUTO" } } } } } } } ] } } } }, "size": SIZE_LIMIT } query = (self.es.domain('swashbucklers') .case_property_query("name", "redbeard") .case_property_query("parrot_name", "polly", clause="should", fuzzy=True)) self.checkQuery(query, json_output) def test_flatten_result(self): expected = {'name': 'blah', 'foo': 'bar', 'baz': 'buzz'} self.assertEqual( flatten_result( { 'name': 'blah', 'case_properties': [ {'key': 'foo', 'value': 'bar'}, {'key': 'baz', 'value': 'buzz'}] } ), expected )