def es_query(self): from corehq.apps.appstore.views import es_query if not getattr(self, 'es_response', None): q = { "query": { "range": { "form.meta.timeEnd": { "from": self.datespan.startdate_param, "to": self.datespan.enddate_param, "include_upper": False}}}, "filter": {"and": []}} xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()]) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) def any_in(a, b): return any(i in b for i in a) if self.request.GET.get('all_mws', 'off') != 'on' or any_in( [str(HQUserType.DEMO_USER), str(HQUserType.ADMIN), str(HQUserType.UNKNOWN)], self.request.GET.getlist('ufilter')): q["filter"]["and"].append( {"terms": {"form.meta.userID": filter(None, self.combined_user_ids)}}) else: ids = filter(None, [user['user_id'] for user in self.get_admins_and_demo_users()]) q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}}) q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}] self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search', start_at=self.pagination.start, size=self.pagination.count) return self.es_response
def inactive_cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.now() then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = { "query": { "bool": { "must_not": { "range": { "modified_on": { "from": then, "to": now } } } } } } data = es_query(params={ "domain.exact": domain, 'closed': False }, q=q, es_url=CASE_INDEX + '/case/_search', size=1) return data['hits']['total'] if data.get('hits') else 0
def cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.now() then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = {"query": {"range": {"modified_on": {"from": then, "to": now}}}} data = es_query(params={"domain.exact": domain, "closed": False}, q=q, es_url=CASE_INDEX + "/case/_search", size=1) return data["hits"]["total"] if data.get("hits") else 0
def _total_users_until_date(dom, date): from corehq.apps.appstore.views import es_query q = { "query": {"term": {"domain": dom}}, "filter": { "and": [ {"range": {"created_on": {"lt": date.strftime('%Y-%m-%d')}}}, {"term": {"doc_type": "CommCareUser"}}, ], }, } return es_query(q=q, es_url=USER_INDEX + '/user/_search')["hits"]["total"]
def es_form_submissions(self, datespan=None, dict_only=False): datespan = datespan or self.datespan q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"range": { "form.meta.timeEnd": { "from": datespan.startdate_param, "to": datespan.enddate_param, "include_upper": True}}}]}}} facets = ['form.meta.userID'] return es_query(q=q, facets=facets, es_url=XFORM_INDEX + '/xform/_search', size=1, dict_only=dict_only)
def es_total_cases(self, datespan=None, dict_only=False): datespan = datespan or self.datespan q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"range": {"opened_on": {"lte": datespan.enddate_param}}}], "must_not": {"range": {"closed_on": {"lt": datespan.startdate_param}}}}}} if self.case_type: q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}}) facets = ['owner_id'] return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def es_q(user_id): q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"match": {"form.meta.userID": user_id}}, {"range": { "form.meta.timeEnd": { "from": datespan.startdate_param, "to": datespan.enddate_param, "include_upper": True}}} ]}}, "sort": {"form.meta.timeEnd" : {"order": "desc"}}} results = es_query(q=q, es_url=XFORM_INDEX + '/xform/_search', size=1, dict_only=dict_only)['hits']['hits'] return results[0]['_source']['form']['meta']['timeEnd'] if results else None
def active_mobile_users(domain, *args): """ Returns the number of mobile users who have submitted a form in the last 30 days """ now = datetime.now() then = (now - timedelta(days=30)).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = {"query": {"range": {"form.meta.timeEnd": {"from": then, "to": now}}}} facets = ["userID"] data = es_query(params={"domain.exact": domain}, q=q, facets=facets, es_url=XFORM_INDEX + "/xform/_search", size=1) excluded_ids = ["commtrack-system"] terms = [t.get("term") for t in data["facets"]["userID"]["terms"]] terms = filter(lambda t: t and t not in excluded_ids, terms) return len(terms)
def inactive_cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.now() then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = {"query": {"bool": { "must_not": { "range": { "modified_on": { "from": then, "to": now }}}}}} data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_url=CASE_INDEX + '/case/_search', size=1) return data['hits']['total'] if data.get('hits') else 0
def es_case_queries(self, date_field, user_field='user_id', datespan=None, dict_only=False): datespan = datespan or self.datespan q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"range": { date_field: { "from": datespan.startdate_param, "to": datespan.enddate_param, "include_upper": True}}} ]}}} if self.case_type: q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}}) facets = [user_field] return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def catch_signal(app, **kwargs): app_name = app.__name__.rsplit(".", 1)[0] if app_name == _module: from corehq.apps.appstore.views import es_query try: q = {"fields": ["name"]} result = es_query(params={"internal.using_call_center": True, "is_active": True, "is_snapshot": False}, q=q) hits = result.get("hits", {}).get("hits", {}) for hit in hits: domain = Domain.get(hit["_id"]) print(" callcenter bootstap `{}`".format(domain.name)) bootstrap_callcenter(domain) except RequestException: if not settings.DEBUG: logging.exception("Unable to query ES for call-center domains during syncdb") else: print("WARNING: unable to query ES for call-center domains", file=sys.stderr)
def active_mobile_users(domain, *args): """ Returns the number of mobile users who have submitted a form in the last 30 days """ now = datetime.now() then = (now - timedelta(days=30)).strftime(DATE_FORMAT) now = now.strftime(DATE_FORMAT) q = {"query": {"range": {"form.meta.timeEnd": {"from": then, "to": now}}}} facets = ['userID'] data = es_query(params={"domain.exact": domain}, q=q, facets=facets, es_url=XFORM_INDEX + '/xform/_search', size=1) excluded_ids = ['commtrack-system'] terms = [t.get('term') for t in data["facets"]["userID"]["terms"]] terms = filter(lambda t: t and t not in excluded_ids, terms) return len(terms)
def es_domain_query(params=None, facets=None, terms=None, domains=None, return_q_dict=False, start_at=None, size=None, sort=None): from corehq.apps.appstore.views import es_query if params is None: params = {} if terms is None: terms = ['search'] if facets is None: facets = [] q = {"query": {"match_all":{}}} if domains is not None: q["query"] = { "in" : { "name" : domains, } } q["filter"] = {"and": [ {"term": {"doc_type": "Domain"}}, {"term": {"is_snapshot": False}}, ]} search_query = params.get('search', "") if search_query: q['query'] = { "bool": { "must": { "match" : { "_all" : { "query" : search_query, "operator" : "or", }}}}} q["facets"] = {} stats = ['cp_n_active_cases', 'cp_n_inactive_cases', 'cp_n_active_cc_users', 'cp_n_cc_users', 'cp_n_60_day_cases', 'cp_n_web_users', 'cp_n_forms', 'cp_n_cases'] for prop in stats: q["facets"].update({"%s-STATS" % prop: {"statistical": {"field": prop}}}) q["sort"] = sort if sort else [{"name" : {"order": "asc"}},] return es_query(params, facets, terms, q, DOMAIN_INDEX + '/hqdomain/_search', start_at, size, dict_only=return_q_dict)
def es_active_cases(self, datespan=None, dict_only=False): """ Open cases that haven't been modified within time range """ datespan = datespan or self.datespan q = {"query": { "bool": { "must": [ {"match": {"domain.exact": self.domain}}, {"nested": { "path": "actions", "query": { "range": { "actions.date": { "from": datespan.startdate_param, "to": datespan.enddate_param, "include_upper": True}}}}}]}}} if self.case_type: q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}}) facets = ['owner_id'] return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def es_domain_query(params, facets=None, terms=None, domains=None, return_q_dict=False, start_at=None, size=None, sort=None): from corehq.apps.appstore.views import es_query if terms is None: terms = ['search'] if facets is None: facets = [] q = {"query": {"match_all":{}}} if domains is not None: q["query"] = { "in" : { "name" : domains, } } q["filter"] = {"and": [ {"term": {"doc_type": "Domain"}}, {"term": {"is_snapshot": False}}, ]} search_query = params.get('search', "") if search_query: q['query'] = { "bool": { "must": { "match" : { "_all" : { "query" : search_query, "operator" : "or", }}}}} q["facets"] = {} stats = ['cp_n_active_cases', 'cp_n_active_cc_users', 'cp_n_cc_users', 'cp_n_web_users', 'cp_n_forms', 'cp_n_cases'] for prop in stats: q["facets"].update({"%s-STATS" % prop: {"statistical": {"field": prop}}}) q["sort"] = sort if sort else [{"name" : {"order": "asc"}},] return es_query(params, facets, terms, q, DOMAIN_INDEX + '/hqdomain/_search', start_at, size, dict_only=return_q_dict)
def es_query(self, params=None): from corehq.apps.appstore.views import es_query if params is None: params = {} terms = ['search'] q = {"query": {"match_all":{}}} search_query = params.get('search', "") if search_query: q['query'] = { "bool": { "must": { "match" : { "_all" : { "query" : search_query, "operator" : "or", }}}}} q["facets"] = {} q["sort"] = self.get_sorting_block() start_at=self.pagination.start size=self.pagination.count return es_query(params, self.es_facet_list, terms, q, self.es_url, start_at, size)