def es_query(self, paginated): reviewed_form_ids = get_form_ids_by_status(self.domain, getattr(self, 'display_status', None)) if len(reviewed_form_ids) > 0: if not getattr(self, 'es_response', None): date_tuple = _get_date_range(self.request_params.get('range', None)) filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1]) location_ids = get_location_hierarchy_by_id( self.request_params.get("location_id", None), self.domain, self.request.couch_user, CCT_only=True ) q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids) if len(reviewed_form_ids) > 0: q["filter"]["and"].append({"ids": {"values": reviewed_form_ids}}) q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}] if paginated: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms', start_at=self.pagination.start, size=self.pagination.count) else: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms') else: self.es_response = {'hits': {'total': 0}} return self.es_response
def es_query(self, paginated): if not getattr(self, 'es_response', None): date_tuple = _get_date_range(self.request_params.get('range', None)) filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1]) location_ids = get_location_hierarchy_by_id( self.request_params.get("location_id", None), self.domain, self.request.couch_user, CCT_only=True ) q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids) xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter")) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) modify_close = [_f for _f in ['Modify/Close Client'] if _f] q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}}) q["sort"] = self.get_sorting_block() \ if self.get_sorting_block() else [{"form.meta.timeEnd": {"order": "desc"}}] if paginated: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms', start_at=self.pagination.start, size=self.pagination.count) else: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms') return self.es_response
def es_query(self, paginated): if not getattr(self, 'es_response', None): date_tuple = _get_date_range(self.request_params.get('range', None)) filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1]) location_ids = get_location_hierarchy_by_id( self.request_params.get("location_id", None), self.domain, self.request.couch_user, CCT_only=True ) q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids) xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter")) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) modify_close = [_f for _f in [u'Modify/Close Client'] if _f] q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}}) q["sort"] = self.get_sorting_block() \ if self.get_sorting_block() else [{"form.meta.timeEnd": {"order": "desc"}}] if paginated: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms', start_at=self.pagination.start, size=self.pagination.count) else: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms') return self.es_response
def selected(self): selected_ids = self.request.GET.getlist(self.slug) if not selected_ids: return [{ 'id': 't__0', 'text': _("[All mobile workers]"), }] basics = dict(self.basics) selected = [] user_ids = [] group_ids = [] for s_id in selected_ids: if s_id in basics: selected.append((s_id, basics.get(s_id))) else: try: kind, key = s_id.split('__') except ValueError: # badly formatted continue if kind == 'u': user_ids.append(key) elif kind == 'g': group_ids.append(key) if group_ids: q = {"query": {"filtered": {"filter": { "ids": {"values": group_ids} }}}} res = es_query( es_url=ES_URLS["groups"], q=q, fields=['_id', 'name'], ) selected += [self.group_tuple(hit['fields']) for hit in res['hits']['hits']] if user_ids: q = {"query": {"filtered": {"filter": { "ids": {"values": user_ids} }}}} res = es_query( es_url=ES_URLS["users"], q=q, fields = ['_id', 'username', 'first_name', 'last_name', 'doc_type'], ) selected += [self.user_tuple(hit['fields']) for hit in res['hits']['hits']] known_ids = dict(selected) return [{'id': id, 'text': known_ids[id]} for id in selected_ids if id in known_ids ]
def selected(self): selected_ids = self.request.GET.getlist(self.slug) if not selected_ids: defaults = [{ 'id': 't__0', 'text': _("[All mobile workers]"), }] if self.request.project.commtrack_enabled: commtrack_tuple = self.user_type_tuple(HQUserType.COMMTRACK) defaults.append({ 'id': commtrack_tuple[0], 'text': commtrack_tuple[1] }) return defaults user_ids = self.selected_user_ids(self.request) user_types = self.selected_user_types(self.request) group_ids = self.selected_group_ids(self.request) selected = [t for t in self.user_types if t[0][3:].isdigit() and int(t[0][3:]) in user_types] if group_ids: q = {"query": {"filtered": {"filter": { "ids": {"values": group_ids} }}}} res = es_query( es_url=ES_URLS["groups"], q=q, fields=['_id', 'name', "case_sharing", "reporting"], ) for group in res['hits']['hits']: if group['fields'].get("reporting", False): selected.append(self.reporting_group_tuple(group['fields'])) if group['fields'].get("case_sharing", False): selected.append(self.sharing_group_tuple(group['fields'])) if user_ids: q = {"query": {"filtered": {"filter": { "ids": {"values": user_ids} }}}} res = es_query( es_url=ES_URLS["users"], q=q, fields = ['_id', 'username', 'first_name', 'last_name', 'doc_type'], ) selected += [self.user_tuple(hit['fields']) for hit in res['hits']['hits']] known_ids = dict(selected) return [{'id': id, 'text': known_ids[id]} for id in selected_ids if id in known_ids ]
def catch_signal(app, **kwargs): app_name = app.__name__.rsplit('.', 1)[0] if app_name == _module: def _log(msg): if not settings.DEBUG: logging.exception(msg) else: print(msg, file=sys.stderr) try: q = {'fields': ['name']} result = es_query(params={ 'internal.using_call_center': True, 'is_active': True, 'is_snapshot': False }, q=q) hits = result.get('hits', {}).get('hits', {}) for hit in hits: try: domain = Domain.get(hit['_id']) print(' callcenter bootstap `{0}`'.format(domain.name)) bootstrap_callcenter(domain) except ResourceNotFound: _log("Couldn't find domain {dom} during call center sync". format(dom=hit['_id'])) except RequestException: _log('Unable to query ES for call-center domains during syncdb')
def options(self): q = { "query": { "filtered": { "query": { "match_all": {} }, "filter": { "and": [ {"term": {"domain.exact": self.domain}}, {"term": {"type.exact": "participant"}}, ] } } } } es_filters = q["query"]["filtered"]["filter"] def _filter_gen(key, flist): return {"terms": { key: [item.lower() for item in flist if item] }} user = self.request.couch_user if not user.is_web_user(): owner_ids = user.get_group_ids() user_ids = [user._id] owner_filters = _filter_gen('owner_id', owner_ids) user_filters = _filter_gen('user_id', user_ids) filters = filter(None, [owner_filters, user_filters]) subterms = [] subterms.append({'or': filters}) es_filters["and"].append({'and': subterms} if subterms else {}) es_results = es_query(q=q, es_url=REPORT_CASE_INDEX + '/_search', dict_only=False) return [(case['_source']['_id'], case['_source']['full_name']['#value']) for case in es_results['hits'].get('hits', [])]
def active_mobile_users(domain, *args): """ Returns the number of mobile users who have submitted a form in the last 30 days """ now = datetime.now() then = (now - timedelta(days=30)).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = { "query": { "range": { "form.meta.timeEnd": { "from": then, "to": now } } }, "filter": { "and": ADD_TO_ES_FILTER["forms"][:] } } facets = ['form.meta.userID'] data = es_query(params={"domain.exact": domain}, q=q, facets=facets, es_url=XFORM_INDEX + '/xform/_search', size=1) terms = [ t.get('term') for t in data["facets"]["form.meta.userID"]["terms"] ] user_ids = CouchUser.ids_by_domain(domain) return len(filter(lambda t: t and t in user_ids, terms))
def inactive_cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.now() then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = { "query": { "bool": { "must_not": { "range": { "modified_on": { "from": then, "to": now } } } } } } data = es_query(params={ "domain.exact": domain, 'closed': False }, q=q, es_url=CASE_INDEX + '/case/_search', size=1) return data['hits']['total'] if data.get('hits') else 0
def es_deployments_query(params, facets=None, terms=None, sort_by="snapshot_time"): if terms is None: terms = ['is_approved', 'sort_by', 'search'] if facets is None: facets = [] q = { "query": { "bool": { "must": [{ "match": { 'doc_type': "Domain" } }, { "term": { "deployment.public": True } }] } } } search_query = params.get('search', "") if search_query: q['query']['bool']['must'].append( {"match": { "_all": { "query": search_query, "operator": "and" } }}) return es_query(params, facets, terms, q)
def es_snapshot_query(params, facets=None, terms=None, sort_by="snapshot_time", start_at=None, size=None): if terms is None: terms = ['is_approved', 'sort_by', 'search'] if facets is None: facets = [] q = {"sort": {sort_by: {"order": "desc"}}, "query": {"bool": {"must": [ {"match": {'doc_type': "Domain"}}, {"term": {"published": True}}, {"term": {"is_snapshot": True}}, {"term": {"snapshot_head": True}} ]}}, "filter": {"and": [{"term": {"is_approved": params.get('is_approved', None) or True}}]}} search_query = params.get('search', "") if search_query: q['query']['bool']['must'].append({ "match": { "_all": { "query": search_query, "operator": "and" } } }) return es_query(params, facets, terms, q, start_at=start_at, size=size)
def es_query(self): if not getattr(self, 'es_response', None): q = { "query": { "range": { "form.meta.timeEnd": { "from": self.datespan.startdate_param, "to": self.datespan.enddate_param, "include_upper": False}}}, "filter": {"and": ADD_TO_ES_FILTER["forms"][:]}} xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()]) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) def any_in(a, b): return any(i in b for i in a) if self.request.GET.get('all_mws', 'off') != 'on' or any_in( [str(HQUserType.DEMO_USER), str(HQUserType.ADMIN), str(HQUserType.UNKNOWN)], self.request.GET.getlist('ufilter')): q["filter"]["and"].append( {"terms": {"form.meta.userID": filter(None, self.combined_user_ids)}}) else: ids = filter(None, [user['user_id'] for user in self.get_admins_and_demo_users()]) q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}}) q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}] self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search', start_at=self.pagination.start, size=self.pagination.count) return self.es_response
def es_query(self): if not getattr(self, 'es_response', None): q = { "query": { "range": { self.time_field: { "from": self.datespan.startdate_param, "to": self.datespan.enddate_param, "include_upper": False}}}, "filter": {"and": ADD_TO_ES_FILTER["forms"][:]}} xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()]) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(self.domain, self.request, True, True) if "t__0" not in self.request.GET.getlist("emw") or users_data["admin_and_demo_users"]: q["filter"]["and"].append( {"terms": {"form.meta.userID": filter(None, [u["user_id"] for u in users_data["combined_users"]])}}) else: negated_ids = util.get_all_users_by_domain(self.domain, user_filter=HQUserType.all_but_users(), simplified=True) ids = filter(None, [user['user_id'] for user in negated_ids]) q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}}) for cp in filter(None, self.request.GET.get('form_data', "").split(",")): q["filter"]["and"].append({"term": {"__props_for_querying": cp.lower()}}) q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{self.time_field : {"order": "desc"}}] self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search', start_at=self.pagination.start, size=self.pagination.count) return self.es_response
def es_results(self): q = { "query": { "filtered": { "query": { }, "filter": { "bool": { "must": [ {"term": {"domain.exact": self.domain}}, ] } } } }, "size": self.pagination.count, "from": self.pagination.start, } es_filters = q["query"]["filtered"]["filter"] if self.awcs: awc_term = get_nested_terms_filter("user_data.awc", self.awcs) es_filters["bool"]["must"].append(awc_term) elif self.gp: gp_term = get_nested_terms_filter("user_data.gp", self.gp) es_filters["bool"]["must"].append(gp_term) elif self.blocks: block_term = get_nested_terms_filter("user_data.block", self.blocks) es_filters["bool"]["must"].append(block_term) q["query"]["filtered"]["query"].update({"match_all": {}}) logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q))) return es_query(q=q, es_url=USER_INDEX + '/_search', dict_only=False, start_at=self.pagination.start, size=self.pagination.count)
def catch_signal(app, **kwargs): app_name = app.__name__.rsplit('.', 1)[0] if app_name == _module: def _log(msg): if not settings.DEBUG: logging.exception(msg) else: print(msg, file=sys.stderr) try: q = {'fields': ['name']} result = es_query(params={ 'internal.using_call_center': True, 'is_active': True, 'is_snapshot': False }, q=q) hits = result.get('hits', {}).get('hits', {}) for hit in hits: try: domain = Domain.get(hit['_id']) print(' callcenter bootstap `{0}`'.format(domain.name)) bootstrap_callcenter(domain) except ResourceNotFound: _log("Couldn't find domain {dom} during call center sync".format(dom=hit['_id'])) except (RequestException, ESError): _log('Unable to query ES for call-center domains during syncdb')
def _total_until_date(histogram_type, datespan, filters=[], domain_list=None): """ Returns the initial values for the non cumulative graphs """ query = { "in": {"domain.exact": domain_list} } if domain_list is not None else {"match_all": {}} q = { "query": query, "filter": { "and": [ { "range": { DATE_FIELDS[histogram_type]: { "lt": datespan.startdate_display } } }, ], }, } q["filter"]["and"].extend(ADD_TO_ES_FILTER.get(histogram_type, [])[:]) q["filter"]["and"].extend(filters) return es_query( q=q, es_url=ES_URLS[histogram_type], size=0, )["hits"]["total"]
def es_query(self, params=None, size=None): if params is None: params = {} terms = ['search'] q = {"query": {"match_all": {}}} search_query = params.get('search', "") if search_query: q['query'] = { "bool": { "must": { "match": { "_all": { "query": search_query, "operator": "and", } } } } } q["facets"] = {} q["sort"] = self.get_sorting_block() start_at = self.pagination.start size = size if size is not None else self.pagination.count return es_query(params, self.es_facet_list, terms, q, self.es_url, start_at, size, facet_size=25)
def appointments_query(self, field_name, facets, check_response_state=False, base_query=None): q = base_query or self.base_query q["filter"] = { "and": [ { "term": { "workflow": WORKFLOW_REMINDER.lower() } }, { "term": { TC_STUB + field_name + '.case_type': 'cc_appointment' } }, ] } if check_response_state: q["filter"]["and"].extend([{ "term": { TC_STUB + field_name + '.response_state': 'confirmed' } }]) return es_query(q=q, es_url=ES_URLS['tc_sms'], facets=facets, size=0)
def inactive_cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.utcnow() then = json_format_datetime(now - timedelta(days=int(days))) now = json_format_datetime(now) q = { "query": { "bool": { "must_not": { "range": { "modified_on": { "from": then, "to": now } } } } } } data = es_query(params={ "domain.exact": domain, 'closed': False }, q=q, es_index='cases', size=1) return data['hits']['total'] if data.get('hits') else 0
def get_rows(self, dataspan): q = { "query": { "filtered": { "query": { }, "filter": { "bool": { "must": [ {"term": {"domain.exact": self.domain}}, ] } } } } } es_filters = q["query"]["filtered"]["filter"] blocks = self.request.GET.getlist('blocks') if blocks: block_lower = [block.lower() for block in blocks] es_filters["bool"]["must"].append({"terms": {"user_data.block": block_lower}}) awcs = self.request.GET.getlist('awcs') if awcs: awcs_lower = [awc.lower() for awc in awcs] awc_term = { "or": [{"and": [{"term": {"user_data.awc": term}} for term in re.split('\s', awc) if not re.search(r'^\W+$', term) or re.search(r'^\w+', term)] } for awc in awcs_lower] } es_filters["bool"]["must"].append(awc_term) q["query"]["filtered"]["query"].update({"match_all": {}}) logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q))) return es_query(q=q, es_url=USER_INDEX + '/_search', dict_only=False)['hits'].get('hits', [])
def total_distinct_users(domains=None): """ Get total number of users who've ever submitted a form. """ query = { "in": { "domain.exact": domains } } if domains is not None else { "match_all": {} } q = { "query": query, "filter": { "and": ADD_TO_ES_FILTER["forms"][:] }, } res = es_query(q=q, facets=["form.meta.userID"], es_index='forms', size=0) user_ids = reduce(list.__add__, [CouchUser.ids_by_domain(d) for d in domains], []) terms = [t.get('term') for t in res["facets"]["form.meta.userID"]["terms"]] return len( filter(lambda t: t and t not in WEIRD_USER_IDS and t in user_ids, terms))
def get_rows(self, dataspan): q = { "query": { "filtered": {"query": {}, "filter": {"bool": {"must": [{"term": {"domain.exact": self.domain}}]}}} } } es_filters = q["query"]["filtered"]["filter"] blocks = self.request.GET.getlist("blocks") if blocks: block_lower = [block.lower() for block in blocks] es_filters["bool"]["must"].append({"terms": {"user_data.block": block_lower}}) awcs = self.request.GET.getlist("awcs") if awcs: awcs_lower = [awc.lower() for awc in awcs] awc_term = { "or": [ { "and": [ {"term": {"user_data.awc": term}} for term in re.split("\s", awc) if not re.search(r"^\W+$", term) or re.search(r"^\w+", term) ] } for awc in awcs_lower ] } es_filters["bool"]["must"].append(awc_term) q["query"]["filtered"]["query"].update({"match_all": {}}) logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q))) return es_query(q=q, es_url=USER_INDEX + "/_search", dict_only=False)["hits"].get("hits", [])
def _total_until_date(histogram_type, datespan, filters=[], domain_list=None): """ Returns the initial values for the non cumulative graphs """ query = { "in": { "domain.exact": domain_list } } if domain_list is not None else { "match_all": {} } q = { "query": query, "filter": { "and": [ { "range": { DATE_FIELDS[histogram_type]: { "lt": datespan.startdate_display } } }, ], }, } q["filter"]["and"].extend(ADD_TO_ES_FILTER.get(histogram_type, [])[:]) q["filter"]["and"].extend(filters) return es_query( q=q, es_index=histogram_type, size=0, )["hits"]["total"]
def query_es(self, limit, skip, query=None): web_user_filter = [ { "term": { "user.domain_memberships.domain": self.domain } }, ] web_user_filter.extend(ADD_TO_ES_FILTER['web_users']) q = { "filter": { "and": web_user_filter }, "sort": { 'username.exact': 'asc' }, } default_fields = ["username", "last_name", "first_name"] q["query"] = search_string_query(query, default_fields) return es_query( params={}, q=q, es_url=ES_URLS["users"], size=limit, start_at=skip, )
def es_results(self): return es_query( params={'domain.exact': self.domain}, q=self.filters_as_es_query(), es_index='forms', start_at=self.pagination.start, size=self.pagination.count, )
def es_query(self, paginated): if not getattr(self, 'es_response', None): range = self.request_params.get('range', None) start_date = None end_date = None if range is not None: dates = str(range).split(_(" to ")) start_date = dates[0] end_date = dates[1] filtered_case_ids = self._get_filtered_cases(start_date, end_date) exclude_form_ids = [ mcct_status.form_id for mcct_status in McctStatus.objects.filter( domain=self.domain, received_on__range=(start_date, end_date) ) if (mcct_status.status != "eligible" or ( not mcct_status.immunized and (date.today() - mcct_status.registration_date).days < 272 and not mcct_status.is_booking and not mcct_status.is_stillbirth)) ] location_ids = get_location_hierarchy_by_id( self.request_params.get("location_id", None), self.domain, self.request.couch_user, CCT_only=True ) q = _get_report_query(start_date, end_date, filtered_case_ids, location_ids) if len(exclude_form_ids) > 0: q["filter"]["and"].append({"not": {"ids": {"values": exclude_form_ids}}}) xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter")) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) modify_close = [_f for _f in [u'Modify/Close Client'] if _f] q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}}) q["sort"] = self.get_sorting_block() \ if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}] if paginated: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms', start_at=self.pagination.start, size=self.pagination.count) else: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms') return self.es_response
def es_results(self): return es_query( params={'domain.exact': self.domain}, q=self.filters_as_es_query(), es_url=XFORM_INDEX + '/xform/_search', start_at=self.pagination.start, size=self.pagination.count, )
def query_es(self, limit, skip, query=None): web_user_filter = [{"term": {"user.domain_memberships.domain": self.domain}}] web_user_filter.extend(ADD_TO_ES_FILTER["web_users"]) q = {"filter": {"and": web_user_filter}, "sort": {"username.exact": "asc"}} default_fields = ["username", "last_name", "first_name"] q["query"] = search_string_query(query, default_fields) return es_query(params={}, q=q, es_url=ES_URLS["users"], size=limit, start_at=skip)
def es_query(self, paginated): if not getattr(self, 'es_response', None): range = self.request_params.get('range', None) start_date = None end_date = None if range is not None: dates = str(range).split(_(" to ")) start_date = dates[0] end_date = dates[1] filtered_case_ids = self._get_filtered_cases(start_date, end_date) exclude_form_ids = [ mcct_status.form_id for mcct_status in McctStatus.objects.filter( domain=self.domain, received_on__range=(start_date, end_date) ) if (mcct_status.status != "eligible" or ( not mcct_status.immunized and (date.today() - mcct_status.registration_date).days < 272 and not mcct_status.is_booking and not mcct_status.is_stillbirth)) ] location_ids = get_location_hierarchy_by_id( self.request_params.get("location_id", None), self.domain, self.request.couch_user, CCT_only=True ) q = _get_report_query(start_date, end_date, filtered_case_ids, location_ids) if len(exclude_form_ids) > 0: q["filter"]["and"].append({"not": {"ids": {"values": exclude_form_ids}}}) xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter")) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) modify_close = [_f for _f in ['Modify/Close Client'] if _f] q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}}) q["sort"] = self.get_sorting_block() \ if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}] if paginated: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms', start_at=self.pagination.start, size=self.pagination.count) else: self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms') return self.es_response
def appointments_query(self, field_name, facets, check_response_state=False, base_query=None): q = base_query or self.base_query q["filter"] = {"and": [ {"term": {"workflow": WORKFLOW_REMINDER.lower()}}, {"term": {TC_STUB+field_name+'.case_type': 'cc_appointment'}}, ]} if check_response_state: q["filter"]["and"].extend([{"term": {TC_STUB+field_name+'.response_state': 'confirmed'}}]) return es_query(q=q, es_url=ES_URLS['tc_sms'], facets=facets, size=0)
def get_call_center_domains(): q = {'fields': ['name']} result = es_query(params={ 'internal.using_call_center': True, 'is_active': True, 'is_snapshot': False }, q=q) hits = result.get('hits', {}).get('hits', {}) return [hit['fields']['name'] for hit in hits]
def query_es(self): q = { "query": {"query_string": {"query": self.query}}, "filter": {"and": ADD_TO_ES_FILTER["users"][:]}, "sort": {"username.exact": "asc"}, } params = {"domain": self.domain, "is_active": not self.show_inactive} self.es_results = es_query( params=params, q=q, es_url=ES_URLS["users"], size=self.users_list_limit, start_at=self.users_list_skip )
def selected_groups_query(self, request): group_ids = self.selected_group_ids(request) if not group_ids: return [] q = {"query": {"filtered": {"filter": {"ids": {"values": group_ids}}}}} return es_query( es_url=ES_URLS["groups"], q=q, fields=['_id', 'name', "case_sharing", "reporting"], )['hits']['hits']
def inactive_cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.now() then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = {"query": {"bool": {"must_not": {"range": {"modified_on": {"from": then, "to": now}}}}}} data = es_query(params={"domain.exact": domain, "closed": False}, q=q, es_url=CASE_INDEX + "/case/_search", size=1) return data["hits"]["total"] if data.get("hits") else 0
def active_query(self, recipient_type): q = self.base_query q["query"]["bool"]["must"].append({"term": {"direction": "i"}}) q["query"]["bool"]["must"].append( {"term": { "couch_recipient_doc_type": recipient_type }}) return es_query(q=q, facets=['couch_recipient'], es_url=ES_URLS['sms'], size=0)
def total_distinct_users(domains=None): """ Get total number of users who've ever submitted a form. """ query = {"in": {"domain.exact": domains}} if domains is not None else {"match_all": {}} q = {"query": query, "filter": {"and": ADD_TO_ES_FILTER["forms"][:]}} res = es_query(q=q, facets=["form.meta.userID"], es_url=ES_URLS["forms"], size=0) user_ids = reduce(list.__add__, [CouchUser.ids_by_domain(d) for d in domains], []) terms = [t.get("term") for t in res["facets"]["form.meta.userID"]["terms"]] return len(filter(lambda t: t and t not in WEIRD_USER_IDS and t in user_ids, terms))
def workflow_query(self, workflow=None, additional_facets=None): additional_facets = additional_facets or [] q = self.base_query if workflow: q["filter"] = {"and": [{"term": {"workflow": workflow.lower()}}]} else: q["filter"] = {"and": [{"not": {"in": {"workflow": [w.lower() for w in WORKFLOWS]}}}]} facets = ['couch_recipient_doc_type', 'direction'] + additional_facets return es_query(q=q, facets=facets, es_url=ES_URLS['sms'], size=0)
def selected_groups_query(self, request): group_ids = self.selected_group_ids(request) if not group_ids: return [] q = {"query": {"filtered": {"filter": { "ids": {"values": group_ids} }}}} return es_query( es_url=ES_URLS["groups"], q=q, fields=['_id', 'name', "case_sharing", "reporting"], )['hits']['hits']
def selected_user_entries(self, request): user_ids = self.selected_user_ids(request) if not user_ids: return [] q = {"query": {"filtered": {"filter": { "ids": {"values": user_ids} }}}} res = es_query( es_url=ES_URLS["users"], q=q, fields=['_id', 'username', 'first_name', 'last_name', 'doc_type'], ) return [self.utils.user_tuple(hit['fields']) for hit in res['hits']['hits']]
def selected_user_entries(self, request): user_ids = self.selected_user_ids(request) if not user_ids: return [] q = {"query": {"filtered": {"filter": {"ids": {"values": user_ids}}}}} res = es_query( es_url=ES_URLS["users"], q=q, fields=['_id', 'username', 'first_name', 'last_name', 'doc_type'], ) return [ self.utils.user_tuple(hit['fields']) for hit in res['hits']['hits'] ]
def get_rows(self, datespan): block = self.block q = { "query": { "filtered": { "query": {"match_all": {}}, "filter": { "bool": { "must": [ {"term": {"domain.exact": self.domain}}, {"term": {"block_name.#value": block.lower()}}, ] } }, } } } es_filters = q["query"]["filtered"]["filter"] if self.snapshot is None: awcs = self.request.GET.getlist("awcs") if awcs: awcs_lower = [awc.lower() for awc in awcs] awc_term = { "or": [ { "and": [ {"term": {"awc_name.#value": term}} for term in re.split("\s", awc) if not re.search(r"^\W+$", term) or re.search(r"^\w+", term) ] } for awc in awcs_lower ] } es_filters["bool"]["must"].append(awc_term) gp = self.request_params.get("gp", None) if gp: users = CouchUser.by_domain(self.domain) users_id = [ user._id for user in users if "user_data" in user and "gp" in user.user_data and user.user_data["gp"] == gp ] es_filters["bool"]["must"].append({"terms": {"owner_id": users_id}}) is_open = self.request_params.get("is_open", None) if is_open: es_filters["bool"]["must"].append({"term": {"closed": is_open == "closed"}}) if self.default_case_type: es_filters["bool"]["must"].append({"term": {"type.exact": self.default_case_type}}) logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q))) return es_query(q=q, es_url=REPORT_CASE_INDEX + "/_search", dict_only=False)["hits"].get("hits", [])
def es_form_submissions(self, datespan=None, dict_only=False): datespan = datespan or self.datespan q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"range": { "form.meta.timeEnd": { "from": datespan.startdate_param, "to": datespan.enddate_param, "include_upper": True}}}]}}} q["filter"] = {"and": ADD_TO_ES_FILTER["forms"][:]} facets = ['form.meta.userID'] return es_query(q=q, facets=facets, es_url=XFORM_INDEX + '/xform/_search', size=1, dict_only=dict_only)
def users_from_es(self): q = { "query": { "query_string": { "query": self.query }}, "filter": {"and": ADD_TO_ES_FILTER["users"][:]}, "sort": {'username.exact': 'asc'}, } params = { "domain": self.domain, "is_active": not self.show_inactive, } results = es_query(params=params, q=q, es_url=ES_URLS["users"], size=self.users_list_limit, start_at=self.users_list_skip) users = [res['_source'] for res in results.get('hits', {}).get('hits', [])] return [CommCareUser.wrap(user) for user in users]
def form_ids_response(self): # returns a list of form_ids # this is called using ReportDispatcher.dispatch(render_as='form_ids', ***) in # the bulk_form_management_async task from corehq.elastic import es_query results = es_query( params={'domain.exact': self.domain}, q=self.filters_as_es_query(), es_index='forms', fields=['_id'], ) form_ids = [res['_id'] for res in results.get('hits', {}).get('hits', [])] return form_ids
def es_total_cases(self, datespan=None, dict_only=False): datespan = datespan or self.datespan q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"range": {"opened_on": {"lte": datespan.enddate_param}}}], "must_not": {"range": {"closed_on": {"lt": datespan.startdate_param}}}}}} if self.case_type: q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}}) facets = ['owner_id'] return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.utcnow() then = json_format_datetime(now - timedelta(days=int(days))) now = json_format_datetime(now) q = {"query": { "range": { "modified_on": { "from": then, "to": now}}}} data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_url=CASE_INDEX + '/case/_search', size=1) return data['hits']['total'] if data.get('hits') else 0
def form_ids_response(self): # returns a list of form_ids # this is called using ReportDispatcher.dispatch(render_as='form_ids', ***) in # the bulk_form_management_async task from corehq.elastic import es_query results = es_query( params={'domain.exact': self.domain}, q=self.filters_as_es_query(), es_index='forms', fields=['_id'], ) form_ids = [ res['_id'] for res in results.get('hits', {}).get('hits', []) ] return form_ids
def options(self): q = { "query": { "filtered": { "query": { "match_all": {} }, "filter": { "and": [ { "term": { "domain.exact": self.domain } }, { "term": { "type.exact": "participant" } }, ] } } } } es_filters = q["query"]["filtered"]["filter"] def _filter_gen(key, flist): return {"terms": {key: [item.lower() for item in flist if item]}} user = self.request.couch_user if not user.is_web_user(): owner_ids = user.get_group_ids() user_ids = [user._id] owner_filters = _filter_gen('owner_id', owner_ids) user_filters = _filter_gen('user_id', user_ids) filters = filter(None, [owner_filters, user_filters]) subterms = [] subterms.append({'or': filters}) es_filters["and"].append({'and': subterms} if subterms else {}) es_results = es_query(q=q, es_url=REPORT_CASE_INDEX + '/_search', dict_only=False) return [(case['_source']['_id'], case['_source']['full_name']['#value']) for case in es_results['hits'].get('hits', [])]
def query_es(self): is_simple, query = smart_query_string(self.query) default_fields = ["username.exact", "last_name", "first_name"] q = { "query": {"query_string": { "query": query, "default_operator": "AND", "fields": default_fields if is_simple else None }}, "filter": {"and": ADD_TO_ES_FILTER["users"][:]}, "sort": {'username.exact': 'asc'}, } params = { "domain": self.domain, "is_active": not self.show_inactive, } self.es_results = es_query(params=params, q=q, es_url=ES_URLS["users"], size=self.users_list_limit, start_at=self.users_list_skip)
def get_rows(self, datespan): block = self.block q = { "query": { "filtered": { "query": { "match_all": {}, }, "filter": { "bool": { "must": [ {"term": {"domain.exact": self.domain}}, {"term": {"block_name.#value": block.lower()}} ] } } } } } es_filters = q["query"]["filtered"]["filter"] if self.snapshot is None: awcs = self.request.GET.getlist('awcs') if awcs: awcs_lower = [awc.lower() for awc in awcs] awc_term = { "or": [{"and": [{"term": {"awc_name.#value": term}} for term in re.split('\s', awc) if not re.search(r'^\W+$', term) or re.search(r'^\w+', term)] } for awc in awcs_lower] } es_filters["bool"]["must"].append(awc_term) gp = self.request_params.get('gp', None) if gp: users = CouchUser.by_domain(self.domain) users_id = [user._id for user in users if 'user_data' in user and 'gp' in user.user_data and user.user_data['gp'] == gp] es_filters["bool"]["must"].append({"terms": {"owner_id": users_id}}) is_open = self.request_params.get('is_open', None) if is_open: es_filters["bool"]["must"].append({"term": {"closed": is_open == 'closed'}}) if self.default_case_type: es_filters["bool"]["must"].append({"term": {"type.exact": self.default_case_type}}) logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q))) return es_query(q=q, es_url=REPORT_CASE_INDEX + '/_search', dict_only=False)['hits'].get('hits', [])
def query_es(self): q = { "filter": { "and": ADD_TO_ES_FILTER["users"][:] }, "sort": { 'username.exact': 'asc' }, } default_fields = ["username.exact", "last_name", "first_name"] q["query"] = search_string_query(self.query, default_fields) params = { "domain": self.domain, "is_active": not self.show_inactive, } self.es_results = es_query(params=params, q=q, es_url=ES_URLS["users"], size=self.users_list_limit, start_at=self.users_list_skip)
def workflow_query(self, workflow=None, additional_facets=None): additional_facets = additional_facets or [] q = self.base_query if workflow: q["filter"] = {"and": [{"term": {"workflow": workflow.lower()}}]} else: q["filter"] = { "and": [{ "not": { "in": { "workflow": [w.lower() for w in WORKFLOWS] } } }] } facets = ['couch_recipient_doc_type', 'direction'] + additional_facets return es_query(q=q, facets=facets, es_url=ES_URLS['sms'], size=0)
def es_results(self): q = { "query": { "filtered": { "query": {}, "filter": { "bool": { "must": [ { "term": { "domain.exact": self.domain } }, ] } } } }, "size": self.pagination.count, "from": self.pagination.start, } es_filters = q["query"]["filtered"]["filter"] if self.awcs: awc_term = get_nested_terms_filter("user_data.awc", self.awcs) es_filters["bool"]["must"].append(awc_term) elif self.gp: gp_term = get_nested_terms_filter("user_data.gp", self.gp) es_filters["bool"]["must"].append(gp_term) elif self.blocks: block_term = get_nested_terms_filter("user_data.block", self.blocks) es_filters["bool"]["must"].append(block_term) q["query"]["filtered"]["query"].update({"match_all": {}}) logging.info( "ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q))) return es_query(q=q, es_url=USER_INDEX + '/_search', dict_only=False, start_at=self.pagination.start, size=self.pagination.count)
def _query_es(limit, skip, query=None): web_user_filter = [ { "term": { "user.domain_memberships.domain": domain } }, { "term": { "doc_type": "WebUser" } }, { "term": { "base_doc": "couchuser" } }, { "term": { "is_active": True } }, ] q = { "filter": { "and": web_user_filter }, "sort": { 'username.exact': 'asc' }, } default_fields = ["username", "last_name", "first_name"] q["query"] = search_string_query(query, default_fields) return es_query( params={}, q=q, es_index='users', size=limit, start_at=skip, )
def _get_filtered_cases(self, start_date, end_date): query = { "query": { "bool": { "must_not": [ {"range": {"modified_on.date": {"lt": start_date}}}, {"range": {"opened_on.date": {"gt": end_date}}} ] } } } case_search = self.request.GET.get("case_search", "") if len(case_search) > 0: query["filter"] = { "and": [ {"regexp": {"name.exact": ".*?%s.*?" % case_search}} ] } es_response = es_query(params={"domain.exact": self.domain}, q=query, es_index='cases') return [res['_source']['_id'] for res in es_response.get('hits', {}).get('hits', [])]
def es_results(self): return es_query( params={'domain.exact': self.domain}, q={ 'query': { 'range': { self.time_field: { 'from': self.datespan.startdate_param, 'to': self.datespan.enddate_param, 'include_upper': False, } } }, 'filter': { 'and': (ADD_TO_ES_FILTER['forms'] + list(self._es_extra_filters())) }, 'sort': self.get_sorting_block(), }, es_url=XFORM_INDEX + '/xform/_search', start_at=self.pagination.start, size=self.pagination.count, )