def es_query(self):
from corehq.apps.appstore.views import es_query
if not getattr(self, 'es_response', None):
q = {
"query": {
"range": {
"form.meta.timeEnd": {
"from": self.datespan.startdate_param,
"to": self.datespan.enddate_param,
"include_upper": False}}},
"filter": {"and": []}}
xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()])
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
def any_in(a, b):
return any(i in b for i in a)
if self.request.GET.get('all_mws', 'off') != 'on' or any_in(
[str(HQUserType.DEMO_USER), str(HQUserType.ADMIN), str(HQUserType.UNKNOWN)],
self.request.GET.getlist('ufilter')):
q["filter"]["and"].append(
{"terms": {"form.meta.userID": filter(None, self.combined_user_ids)}})
else:
ids = filter(None, [user['user_id'] for user in self.get_admins_and_demo_users()])
q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}})
q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}]
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search',
start_at=self.pagination.start, size=self.pagination.count)
return self.es_response
def inactive_cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.now()
then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {
"query": {
"bool": {
"must_not": {
"range": {
"modified_on": {
"from": then,
"to": now
}
}
}
}
}
}
data = es_query(params={
"domain.exact": domain,
'closed': False
},
q=q,
es_url=CASE_INDEX + '/case/_search',
size=1)
return data['hits']['total'] if data.get('hits') else 0
def cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.now()
then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {"query": {"range": {"modified_on": {"from": then, "to": now}}}}
data = es_query(params={"domain.exact": domain, "closed": False}, q=q, es_url=CASE_INDEX + "/case/_search", size=1)
return data["hits"]["total"] if data.get("hits") else 0
def _total_users_until_date(dom, date):
from corehq.apps.appstore.views import es_query
q = {
"query": {"term": {"domain": dom}},
"filter": {
"and": [
{"range": {"created_on": {"lt": date.strftime('%Y-%m-%d')}}},
{"term": {"doc_type": "CommCareUser"}},
],
},
}
return es_query(q=q, es_url=USER_INDEX + '/user/_search')["hits"]["total"]
def es_form_submissions(self, datespan=None, dict_only=False):
datespan = datespan or self.datespan
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"range": {
"form.meta.timeEnd": {
"from": datespan.startdate_param,
"to": datespan.enddate_param,
"include_upper": True}}}]}}}
facets = ['form.meta.userID']
return es_query(q=q, facets=facets, es_url=XFORM_INDEX + '/xform/_search', size=1, dict_only=dict_only)
def es_total_cases(self, datespan=None, dict_only=False):
datespan = datespan or self.datespan
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"range": {"opened_on": {"lte": datespan.enddate_param}}}],
"must_not": {"range": {"closed_on": {"lt": datespan.startdate_param}}}}}}
if self.case_type:
q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}})
facets = ['owner_id']
return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def es_q(user_id):
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"match": {"form.meta.userID": user_id}},
{"range": {
"form.meta.timeEnd": {
"from": datespan.startdate_param,
"to": datespan.enddate_param,
"include_upper": True}}}
]}},
"sort": {"form.meta.timeEnd" : {"order": "desc"}}}
results = es_query(q=q, es_url=XFORM_INDEX + '/xform/_search', size=1, dict_only=dict_only)['hits']['hits']
return results[0]['_source']['form']['meta']['timeEnd'] if results else None
def active_mobile_users(domain, *args):
"""
Returns the number of mobile users who have submitted a form in the last 30 days
"""
now = datetime.now()
then = (now - timedelta(days=30)).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {"query": {"range": {"form.meta.timeEnd": {"from": then, "to": now}}}}
facets = ["userID"]
data = es_query(params={"domain.exact": domain}, q=q, facets=facets, es_url=XFORM_INDEX + "/xform/_search", size=1)
excluded_ids = ["commtrack-system"]
terms = [t.get("term") for t in data["facets"]["userID"]["terms"]]
terms = filter(lambda t: t and t not in excluded_ids, terms)
return len(terms)
def inactive_cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.now()
then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {"query":
{"bool": {
"must_not": {
"range": {
"modified_on": {
"from": then,
"to": now }}}}}}
data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_url=CASE_INDEX + '/case/_search', size=1)
return data['hits']['total'] if data.get('hits') else 0
def es_case_queries(self, date_field, user_field='user_id', datespan=None, dict_only=False):
datespan = datespan or self.datespan
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"range": {
date_field: {
"from": datespan.startdate_param,
"to": datespan.enddate_param,
"include_upper": True}}}
]}}}
if self.case_type:
q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}})
facets = [user_field]
return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def catch_signal(app, **kwargs):
app_name = app.__name__.rsplit(".", 1)[0]
if app_name == _module:
from corehq.apps.appstore.views import es_query
try:
q = {"fields": ["name"]}
result = es_query(params={"internal.using_call_center": True, "is_active": True, "is_snapshot": False}, q=q)
hits = result.get("hits", {}).get("hits", {})
for hit in hits:
domain = Domain.get(hit["_id"])
print(" callcenter bootstap `{}`".format(domain.name))
bootstrap_callcenter(domain)
except RequestException:
if not settings.DEBUG:
logging.exception("Unable to query ES for call-center domains during syncdb")
else:
print("WARNING: unable to query ES for call-center domains", file=sys.stderr)
def active_mobile_users(domain, *args):
"""
Returns the number of mobile users who have submitted a form in the last 30 days
"""
now = datetime.now()
then = (now - timedelta(days=30)).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {"query": {"range": {"form.meta.timeEnd": {"from": then, "to": now}}}}
facets = ['userID']
data = es_query(params={"domain.exact": domain},
q=q,
facets=facets,
es_url=XFORM_INDEX + '/xform/_search',
size=1)
excluded_ids = ['commtrack-system']
terms = [t.get('term') for t in data["facets"]["userID"]["terms"]]
terms = filter(lambda t: t and t not in excluded_ids, terms)
return len(terms)
def es_domain_query(params=None, facets=None, terms=None, domains=None, return_q_dict=False, start_at=None, size=None, sort=None):
from corehq.apps.appstore.views import es_query
if params is None:
params = {}
if terms is None:
terms = ['search']
if facets is None:
facets = []
q = {"query": {"match_all":{}}}
if domains is not None:
q["query"] = {
"in" : {
"name" : domains,
}
}
q["filter"] = {"and": [
{"term": {"doc_type": "Domain"}},
{"term": {"is_snapshot": False}},
]}
search_query = params.get('search', "")
if search_query:
q['query'] = {
"bool": {
"must": {
"match" : {
"_all" : {
"query" : search_query,
"operator" : "or", }}}}}
q["facets"] = {}
stats = ['cp_n_active_cases', 'cp_n_inactive_cases', 'cp_n_active_cc_users', 'cp_n_cc_users', 'cp_n_60_day_cases', 'cp_n_web_users', 'cp_n_forms', 'cp_n_cases']
for prop in stats:
q["facets"].update({"%s-STATS" % prop: {"statistical": {"field": prop}}})
q["sort"] = sort if sort else [{"name" : {"order": "asc"}},]
return es_query(params, facets, terms, q, DOMAIN_INDEX + '/hqdomain/_search', start_at, size, dict_only=return_q_dict)
def es_active_cases(self, datespan=None, dict_only=False):
"""
Open cases that haven't been modified within time range
"""
datespan = datespan or self.datespan
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"nested": {
"path": "actions",
"query": {
"range": {
"actions.date": {
"from": datespan.startdate_param,
"to": datespan.enddate_param,
"include_upper": True}}}}}]}}}
if self.case_type:
q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}})
facets = ['owner_id']
return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def es_domain_query(params, facets=None, terms=None, domains=None, return_q_dict=False, start_at=None, size=None, sort=None):
from corehq.apps.appstore.views import es_query
if terms is None:
terms = ['search']
if facets is None:
facets = []
q = {"query": {"match_all":{}}}
if domains is not None:
q["query"] = {
"in" : {
"name" : domains,
}
}
q["filter"] = {"and": [
{"term": {"doc_type": "Domain"}},
{"term": {"is_snapshot": False}},
]}
search_query = params.get('search', "")
if search_query:
q['query'] = {
"bool": {
"must": {
"match" : {
"_all" : {
"query" : search_query,
"operator" : "or", }}}}}
q["facets"] = {}
stats = ['cp_n_active_cases', 'cp_n_active_cc_users', 'cp_n_cc_users', 'cp_n_web_users', 'cp_n_forms', 'cp_n_cases']
for prop in stats:
q["facets"].update({"%s-STATS" % prop: {"statistical": {"field": prop}}})
q["sort"] = sort if sort else [{"name" : {"order": "asc"}},]
return es_query(params, facets, terms, q, DOMAIN_INDEX + '/hqdomain/_search', start_at, size, dict_only=return_q_dict)
def es_query(self, params=None):
from corehq.apps.appstore.views import es_query
if params is None:
params = {}
terms = ['search']
q = {"query": {"match_all":{}}}
search_query = params.get('search', "")
if search_query:
q['query'] = {
"bool": {
"must": {
"match" : {
"_all" : {
"query" : search_query,
"operator" : "or", }}}}}
q["facets"] = {}
q["sort"] = self.get_sorting_block()
start_at=self.pagination.start
size=self.pagination.count
return es_query(params, self.es_facet_list, terms, q, self.es_url, start_at, size)