def es_query(self, paginated):
reviewed_form_ids = get_form_ids_by_status(self.domain, getattr(self, 'display_status', None))
if len(reviewed_form_ids) > 0:
if not getattr(self, 'es_response', None):
date_tuple = _get_date_range(self.request_params.get('range', None))
filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1])
location_ids = get_location_hierarchy_by_id(
self.request_params.get("location_id", None),
self.domain,
self.request.couch_user,
CCT_only=True
)
q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids)
if len(reviewed_form_ids) > 0:
q["filter"]["and"].append({"ids": {"values": reviewed_form_ids}})
q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}]
if paginated:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms',
start_at=self.pagination.start, size=self.pagination.count)
else:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms')
else:
self.es_response = {'hits': {'total': 0}}
return self.es_response
def es_query(self, paginated):
reviewed_form_ids = get_form_ids_by_status(self.domain, getattr(self, 'display_status', None))
if len(reviewed_form_ids) > 0:
if not getattr(self, 'es_response', None):
date_tuple = _get_date_range(self.request_params.get('range', None))
filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1])
location_ids = get_location_hierarchy_by_id(
self.request_params.get("location_id", None),
self.domain,
self.request.couch_user,
CCT_only=True
)
q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids)
if len(reviewed_form_ids) > 0:
q["filter"]["and"].append({"ids": {"values": reviewed_form_ids}})
q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}]
if paginated:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms',
start_at=self.pagination.start, size=self.pagination.count)
else:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms')
else:
self.es_response = {'hits': {'total': 0}}
return self.es_response
def es_query(self, paginated):
if not getattr(self, 'es_response', None):
date_tuple = _get_date_range(self.request_params.get('range', None))
filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1])
location_ids = get_location_hierarchy_by_id(
self.request_params.get("location_id", None),
self.domain,
self.request.couch_user,
CCT_only=True
)
q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids)
xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter"))
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
modify_close = [_f for _f in ['Modify/Close Client'] if _f]
q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}})
q["sort"] = self.get_sorting_block() \
if self.get_sorting_block() else [{"form.meta.timeEnd": {"order": "desc"}}]
if paginated:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms',
start_at=self.pagination.start, size=self.pagination.count)
else:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms')
return self.es_response
def es_query(self, paginated):
if not getattr(self, 'es_response', None):
date_tuple = _get_date_range(self.request_params.get('range', None))
filtered_case_ids = self._get_filtered_cases(date_tuple[0], date_tuple[1])
location_ids = get_location_hierarchy_by_id(
self.request_params.get("location_id", None),
self.domain,
self.request.couch_user,
CCT_only=True
)
q = _get_report_query(date_tuple[0], date_tuple[1], filtered_case_ids, location_ids)
xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter"))
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
modify_close = [_f for _f in [u'Modify/Close Client'] if _f]
q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}})
q["sort"] = self.get_sorting_block() \
if self.get_sorting_block() else [{"form.meta.timeEnd": {"order": "desc"}}]
if paginated:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms',
start_at=self.pagination.start, size=self.pagination.count)
else:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms')
return self.es_response
def selected(self):
selected_ids = self.request.GET.getlist(self.slug)
if not selected_ids:
return [{
'id': 't__0',
'text': _("[All mobile workers]"),
}]
basics = dict(self.basics)
selected = []
user_ids = []
group_ids = []
for s_id in selected_ids:
if s_id in basics:
selected.append((s_id, basics.get(s_id)))
else:
try:
kind, key = s_id.split('__')
except ValueError:
# badly formatted
continue
if kind == 'u':
user_ids.append(key)
elif kind == 'g':
group_ids.append(key)
if group_ids:
q = {"query": {"filtered": {"filter": {
"ids": {"values": group_ids}
}}}}
res = es_query(
es_url=ES_URLS["groups"],
q=q,
fields=['_id', 'name'],
)
selected += [self.group_tuple(hit['fields']) for hit in res['hits']['hits']]
if user_ids:
q = {"query": {"filtered": {"filter": {
"ids": {"values": user_ids}
}}}}
res = es_query(
es_url=ES_URLS["users"],
q=q,
fields = ['_id', 'username', 'first_name', 'last_name', 'doc_type'],
)
selected += [self.user_tuple(hit['fields']) for hit in res['hits']['hits']]
known_ids = dict(selected)
return [{'id': id, 'text': known_ids[id]}
for id in selected_ids
if id in known_ids
]
def selected(self):
selected_ids = self.request.GET.getlist(self.slug)
if not selected_ids:
defaults = [{
'id': 't__0',
'text': _("[All mobile workers]"),
}]
if self.request.project.commtrack_enabled:
commtrack_tuple = self.user_type_tuple(HQUserType.COMMTRACK)
defaults.append({
'id': commtrack_tuple[0],
'text': commtrack_tuple[1]
})
return defaults
user_ids = self.selected_user_ids(self.request)
user_types = self.selected_user_types(self.request)
group_ids = self.selected_group_ids(self.request)
selected = [t for t in self.user_types
if t[0][3:].isdigit() and int(t[0][3:]) in user_types]
if group_ids:
q = {"query": {"filtered": {"filter": {
"ids": {"values": group_ids}
}}}}
res = es_query(
es_url=ES_URLS["groups"],
q=q,
fields=['_id', 'name', "case_sharing", "reporting"],
)
for group in res['hits']['hits']:
if group['fields'].get("reporting", False):
selected.append(self.reporting_group_tuple(group['fields']))
if group['fields'].get("case_sharing", False):
selected.append(self.sharing_group_tuple(group['fields']))
if user_ids:
q = {"query": {"filtered": {"filter": {
"ids": {"values": user_ids}
}}}}
res = es_query(
es_url=ES_URLS["users"],
q=q,
fields = ['_id', 'username', 'first_name', 'last_name', 'doc_type'],
)
selected += [self.user_tuple(hit['fields']) for hit in res['hits']['hits']]
known_ids = dict(selected)
return [{'id': id, 'text': known_ids[id]}
for id in selected_ids
if id in known_ids
]
def catch_signal(app, **kwargs):
app_name = app.__name__.rsplit('.', 1)[0]
if app_name == _module:
def _log(msg):
if not settings.DEBUG:
logging.exception(msg)
else:
print(msg, file=sys.stderr)
try:
q = {'fields': ['name']}
result = es_query(params={
'internal.using_call_center': True,
'is_active': True,
'is_snapshot': False
},
q=q)
hits = result.get('hits', {}).get('hits', {})
for hit in hits:
try:
domain = Domain.get(hit['_id'])
print(' callcenter bootstap `{0}`'.format(domain.name))
bootstrap_callcenter(domain)
except ResourceNotFound:
_log("Couldn't find domain {dom} during call center sync".
format(dom=hit['_id']))
except RequestException:
_log('Unable to query ES for call-center domains during syncdb')
def options(self):
q = { "query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [
{"term": {"domain.exact": self.domain}},
{"term": {"type.exact": "participant"}},
]
}
}
}
}
es_filters = q["query"]["filtered"]["filter"]
def _filter_gen(key, flist):
return {"terms": {
key: [item.lower() for item in flist if item]
}}
user = self.request.couch_user
if not user.is_web_user():
owner_ids = user.get_group_ids()
user_ids = [user._id]
owner_filters = _filter_gen('owner_id', owner_ids)
user_filters = _filter_gen('user_id', user_ids)
filters = filter(None, [owner_filters, user_filters])
subterms = []
subterms.append({'or': filters})
es_filters["and"].append({'and': subterms} if subterms else {})
es_results = es_query(q=q, es_url=REPORT_CASE_INDEX + '/_search', dict_only=False)
return [(case['_source']['_id'], case['_source']['full_name']['#value']) for case in es_results['hits'].get('hits', [])]
def active_mobile_users(domain, *args):
"""
Returns the number of mobile users who have submitted a form in the last 30 days
"""
now = datetime.now()
then = (now - timedelta(days=30)).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {
"query": {
"range": {
"form.meta.timeEnd": {
"from": then,
"to": now
}
}
},
"filter": {
"and": ADD_TO_ES_FILTER["forms"][:]
}
}
facets = ['form.meta.userID']
data = es_query(params={"domain.exact": domain},
q=q,
facets=facets,
es_url=XFORM_INDEX + '/xform/_search',
size=1)
terms = [
t.get('term') for t in data["facets"]["form.meta.userID"]["terms"]
]
user_ids = CouchUser.ids_by_domain(domain)
return len(filter(lambda t: t and t in user_ids, terms))
def inactive_cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.now()
then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {
"query": {
"bool": {
"must_not": {
"range": {
"modified_on": {
"from": then,
"to": now
}
}
}
}
}
}
data = es_query(params={
"domain.exact": domain,
'closed': False
},
q=q,
es_url=CASE_INDEX + '/case/_search',
size=1)
return data['hits']['total'] if data.get('hits') else 0
def es_deployments_query(params,
facets=None,
terms=None,
sort_by="snapshot_time"):
if terms is None:
terms = ['is_approved', 'sort_by', 'search']
if facets is None:
facets = []
q = {
"query": {
"bool": {
"must": [{
"match": {
'doc_type': "Domain"
}
}, {
"term": {
"deployment.public": True
}
}]
}
}
}
search_query = params.get('search', "")
if search_query:
q['query']['bool']['must'].append(
{"match": {
"_all": {
"query": search_query,
"operator": "and"
}
}})
return es_query(params, facets, terms, q)
def es_snapshot_query(params, facets=None, terms=None, sort_by="snapshot_time", start_at=None, size=None):
if terms is None:
terms = ['is_approved', 'sort_by', 'search']
if facets is None:
facets = []
q = {"sort": {sort_by: {"order": "desc"}},
"query": {"bool": {"must": [
{"match": {'doc_type': "Domain"}},
{"term": {"published": True}},
{"term": {"is_snapshot": True}},
{"term": {"snapshot_head": True}}
]}},
"filter": {"and": [{"term": {"is_approved": params.get('is_approved', None) or True}}]}}
search_query = params.get('search', "")
if search_query:
q['query']['bool']['must'].append({
"match": {
"_all": {
"query": search_query,
"operator": "and"
}
}
})
return es_query(params, facets, terms, q, start_at=start_at, size=size)
def es_query(self):
if not getattr(self, 'es_response', None):
q = {
"query": {
"range": {
"form.meta.timeEnd": {
"from": self.datespan.startdate_param,
"to": self.datespan.enddate_param,
"include_upper": False}}},
"filter": {"and": ADD_TO_ES_FILTER["forms"][:]}}
xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()])
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
def any_in(a, b):
return any(i in b for i in a)
if self.request.GET.get('all_mws', 'off') != 'on' or any_in(
[str(HQUserType.DEMO_USER), str(HQUserType.ADMIN), str(HQUserType.UNKNOWN)],
self.request.GET.getlist('ufilter')):
q["filter"]["and"].append(
{"terms": {"form.meta.userID": filter(None, self.combined_user_ids)}})
else:
ids = filter(None, [user['user_id'] for user in self.get_admins_and_demo_users()])
q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}})
q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}]
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search',
start_at=self.pagination.start, size=self.pagination.count)
return self.es_response
def es_query(self):
if not getattr(self, 'es_response', None):
q = {
"query": {
"range": {
self.time_field: {
"from": self.datespan.startdate_param,
"to": self.datespan.enddate_param,
"include_upper": False}}},
"filter": {"and": ADD_TO_ES_FILTER["forms"][:]}}
xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()])
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(self.domain, self.request, True, True)
if "t__0" not in self.request.GET.getlist("emw") or users_data["admin_and_demo_users"]:
q["filter"]["and"].append(
{"terms": {"form.meta.userID": filter(None, [u["user_id"] for u in users_data["combined_users"]])}})
else:
negated_ids = util.get_all_users_by_domain(self.domain, user_filter=HQUserType.all_but_users(), simplified=True)
ids = filter(None, [user['user_id'] for user in negated_ids])
q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}})
for cp in filter(None, self.request.GET.get('form_data', "").split(",")):
q["filter"]["and"].append({"term": {"__props_for_querying": cp.lower()}})
q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{self.time_field : {"order": "desc"}}]
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search',
start_at=self.pagination.start, size=self.pagination.count)
return self.es_response
def es_results(self):
q = {
"query": {
"filtered": {
"query": {
},
"filter": {
"bool": {
"must": [
{"term": {"domain.exact": self.domain}},
]
}
}
}
},
"size": self.pagination.count,
"from": self.pagination.start,
}
es_filters = q["query"]["filtered"]["filter"]
if self.awcs:
awc_term = get_nested_terms_filter("user_data.awc", self.awcs)
es_filters["bool"]["must"].append(awc_term)
elif self.gp:
gp_term = get_nested_terms_filter("user_data.gp", self.gp)
es_filters["bool"]["must"].append(gp_term)
elif self.blocks:
block_term = get_nested_terms_filter("user_data.block", self.blocks)
es_filters["bool"]["must"].append(block_term)
q["query"]["filtered"]["query"].update({"match_all": {}})
logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q)))
return es_query(q=q, es_url=USER_INDEX + '/_search', dict_only=False,
start_at=self.pagination.start, size=self.pagination.count)
def catch_signal(app, **kwargs):
app_name = app.__name__.rsplit('.', 1)[0]
if app_name == _module:
def _log(msg):
if not settings.DEBUG:
logging.exception(msg)
else:
print(msg, file=sys.stderr)
try:
q = {'fields': ['name']}
result = es_query(params={
'internal.using_call_center': True,
'is_active': True,
'is_snapshot': False
}, q=q)
hits = result.get('hits', {}).get('hits', {})
for hit in hits:
try:
domain = Domain.get(hit['_id'])
print(' callcenter bootstap `{0}`'.format(domain.name))
bootstrap_callcenter(domain)
except ResourceNotFound:
_log("Couldn't find domain {dom} during call center sync".format(dom=hit['_id']))
except (RequestException, ESError):
_log('Unable to query ES for call-center domains during syncdb')
def _total_until_date(histogram_type, datespan, filters=[], domain_list=None):
"""
Returns the initial values for the non cumulative graphs
"""
query = {
"in": {"domain.exact": domain_list}
} if domain_list is not None else {"match_all": {}}
q = {
"query": query,
"filter": {
"and": [
{
"range": {
DATE_FIELDS[histogram_type]: {
"lt": datespan.startdate_display
}
}
},
],
},
}
q["filter"]["and"].extend(ADD_TO_ES_FILTER.get(histogram_type, [])[:])
q["filter"]["and"].extend(filters)
return es_query(
q=q,
es_url=ES_URLS[histogram_type],
size=0,
)["hits"]["total"]
def es_query(self, params=None, size=None):
if params is None:
params = {}
terms = ['search']
q = {"query": {"match_all": {}}}
search_query = params.get('search', "")
if search_query:
q['query'] = {
"bool": {
"must": {
"match": {
"_all": {
"query": search_query,
"operator": "and",
}
}
}
}
}
q["facets"] = {}
q["sort"] = self.get_sorting_block()
start_at = self.pagination.start
size = size if size is not None else self.pagination.count
return es_query(params,
self.es_facet_list,
terms,
q,
self.es_url,
start_at,
size,
facet_size=25)
def appointments_query(self,
field_name,
facets,
check_response_state=False,
base_query=None):
q = base_query or self.base_query
q["filter"] = {
"and": [
{
"term": {
"workflow": WORKFLOW_REMINDER.lower()
}
},
{
"term": {
TC_STUB + field_name + '.case_type': 'cc_appointment'
}
},
]
}
if check_response_state:
q["filter"]["and"].extend([{
"term": {
TC_STUB + field_name + '.response_state': 'confirmed'
}
}])
return es_query(q=q, es_url=ES_URLS['tc_sms'], facets=facets, size=0)
def inactive_cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.utcnow()
then = json_format_datetime(now - timedelta(days=int(days)))
now = json_format_datetime(now)
q = {
"query": {
"bool": {
"must_not": {
"range": {
"modified_on": {
"from": then,
"to": now
}
}
}
}
}
}
data = es_query(params={
"domain.exact": domain,
'closed': False
},
q=q,
es_index='cases',
size=1)
return data['hits']['total'] if data.get('hits') else 0
def get_rows(self, dataspan):
q = {
"query": {
"filtered": {
"query": {
},
"filter": {
"bool": {
"must": [
{"term": {"domain.exact": self.domain}},
]
}
}
}
}
}
es_filters = q["query"]["filtered"]["filter"]
blocks = self.request.GET.getlist('blocks')
if blocks:
block_lower = [block.lower() for block in blocks]
es_filters["bool"]["must"].append({"terms": {"user_data.block": block_lower}})
awcs = self.request.GET.getlist('awcs')
if awcs:
awcs_lower = [awc.lower() for awc in awcs]
awc_term = {
"or":
[{"and": [{"term": {"user_data.awc": term}} for term in re.split('\s', awc)
if not re.search(r'^\W+$', term) or re.search(r'^\w+', term)]
} for awc in awcs_lower]
}
es_filters["bool"]["must"].append(awc_term)
q["query"]["filtered"]["query"].update({"match_all": {}})
logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q)))
return es_query(q=q, es_url=USER_INDEX + '/_search', dict_only=False)['hits'].get('hits', [])
def total_distinct_users(domains=None):
"""
Get total number of users who've ever submitted a form.
"""
query = {
"in": {
"domain.exact": domains
}
} if domains is not None else {
"match_all": {}
}
q = {
"query": query,
"filter": {
"and": ADD_TO_ES_FILTER["forms"][:]
},
}
res = es_query(q=q, facets=["form.meta.userID"], es_index='forms', size=0)
user_ids = reduce(list.__add__,
[CouchUser.ids_by_domain(d) for d in domains], [])
terms = [t.get('term') for t in res["facets"]["form.meta.userID"]["terms"]]
return len(
filter(lambda t: t and t not in WEIRD_USER_IDS and t in user_ids,
terms))
def get_rows(self, dataspan):
q = {
"query": {
"filtered": {"query": {}, "filter": {"bool": {"must": [{"term": {"domain.exact": self.domain}}]}}}
}
}
es_filters = q["query"]["filtered"]["filter"]
blocks = self.request.GET.getlist("blocks")
if blocks:
block_lower = [block.lower() for block in blocks]
es_filters["bool"]["must"].append({"terms": {"user_data.block": block_lower}})
awcs = self.request.GET.getlist("awcs")
if awcs:
awcs_lower = [awc.lower() for awc in awcs]
awc_term = {
"or": [
{
"and": [
{"term": {"user_data.awc": term}}
for term in re.split("\s", awc)
if not re.search(r"^\W+$", term) or re.search(r"^\w+", term)
]
}
for awc in awcs_lower
]
}
es_filters["bool"]["must"].append(awc_term)
q["query"]["filtered"]["query"].update({"match_all": {}})
logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q)))
return es_query(q=q, es_url=USER_INDEX + "/_search", dict_only=False)["hits"].get("hits", [])
def _total_until_date(histogram_type, datespan, filters=[], domain_list=None):
"""
Returns the initial values for the non cumulative graphs
"""
query = {
"in": {
"domain.exact": domain_list
}
} if domain_list is not None else {
"match_all": {}
}
q = {
"query": query,
"filter": {
"and": [
{
"range": {
DATE_FIELDS[histogram_type]: {
"lt": datespan.startdate_display
}
}
},
],
},
}
q["filter"]["and"].extend(ADD_TO_ES_FILTER.get(histogram_type, [])[:])
q["filter"]["and"].extend(filters)
return es_query(
q=q,
es_index=histogram_type,
size=0,
)["hits"]["total"]
def query_es(self, limit, skip, query=None):
web_user_filter = [
{
"term": {
"user.domain_memberships.domain": self.domain
}
},
]
web_user_filter.extend(ADD_TO_ES_FILTER['web_users'])
q = {
"filter": {
"and": web_user_filter
},
"sort": {
'username.exact': 'asc'
},
}
default_fields = ["username", "last_name", "first_name"]
q["query"] = search_string_query(query, default_fields)
return es_query(
params={},
q=q,
es_url=ES_URLS["users"],
size=limit,
start_at=skip,
)
def es_snapshot_query(params, facets=None, terms=None, sort_by="snapshot_time", start_at=None, size=None):
if terms is None:
terms = ['is_approved', 'sort_by', 'search']
if facets is None:
facets = []
q = {"sort": {sort_by: {"order": "desc"}},
"query": {"bool": {"must": [
{"match": {'doc_type': "Domain"}},
{"term": {"published": True}},
{"term": {"is_snapshot": True}},
{"term": {"snapshot_head": True}}
]}},
"filter": {"and": [{"term": {"is_approved": params.get('is_approved', None) or True}}]}}
search_query = params.get('search', "")
if search_query:
q['query']['bool']['must'].append({
"match": {
"_all": {
"query": search_query,
"operator": "and"
}
}
})
return es_query(params, facets, terms, q, start_at=start_at, size=size)
def es_results(self):
return es_query(
params={'domain.exact': self.domain},
q=self.filters_as_es_query(),
es_index='forms',
start_at=self.pagination.start,
size=self.pagination.count,
)
def es_query(self, paginated):
if not getattr(self, 'es_response', None):
range = self.request_params.get('range', None)
start_date = None
end_date = None
if range is not None:
dates = str(range).split(_(" to "))
start_date = dates[0]
end_date = dates[1]
filtered_case_ids = self._get_filtered_cases(start_date, end_date)
exclude_form_ids = [
mcct_status.form_id
for mcct_status in McctStatus.objects.filter(
domain=self.domain,
received_on__range=(start_date, end_date)
)
if (mcct_status.status != "eligible" or (
not mcct_status.immunized and (date.today() - mcct_status.registration_date).days < 272 and
not mcct_status.is_booking and not mcct_status.is_stillbirth))
]
location_ids = get_location_hierarchy_by_id(
self.request_params.get("location_id", None),
self.domain,
self.request.couch_user,
CCT_only=True
)
q = _get_report_query(start_date, end_date, filtered_case_ids, location_ids)
if len(exclude_form_ids) > 0:
q["filter"]["and"].append({"not": {"ids": {"values": exclude_form_ids}}})
xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter"))
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
modify_close = [_f for _f in [u'Modify/Close Client'] if _f]
q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}})
q["sort"] = self.get_sorting_block() \
if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}]
if paginated:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms',
start_at=self.pagination.start, size=self.pagination.count)
else:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms')
return self.es_response
def es_results(self):
return es_query(
params={'domain.exact': self.domain},
q=self.filters_as_es_query(),
es_index='forms',
start_at=self.pagination.start,
size=self.pagination.count,
)
def es_results(self):
return es_query(
params={'domain.exact': self.domain},
q=self.filters_as_es_query(),
es_url=XFORM_INDEX + '/xform/_search',
start_at=self.pagination.start,
size=self.pagination.count,
)
def query_es(self, limit, skip, query=None):
web_user_filter = [{"term": {"user.domain_memberships.domain": self.domain}}]
web_user_filter.extend(ADD_TO_ES_FILTER["web_users"])
q = {"filter": {"and": web_user_filter}, "sort": {"username.exact": "asc"}}
default_fields = ["username", "last_name", "first_name"]
q["query"] = search_string_query(query, default_fields)
return es_query(params={}, q=q, es_url=ES_URLS["users"], size=limit, start_at=skip)
def es_query(self, paginated):
if not getattr(self, 'es_response', None):
range = self.request_params.get('range', None)
start_date = None
end_date = None
if range is not None:
dates = str(range).split(_(" to "))
start_date = dates[0]
end_date = dates[1]
filtered_case_ids = self._get_filtered_cases(start_date, end_date)
exclude_form_ids = [
mcct_status.form_id
for mcct_status in McctStatus.objects.filter(
domain=self.domain,
received_on__range=(start_date, end_date)
)
if (mcct_status.status != "eligible" or (
not mcct_status.immunized and (date.today() - mcct_status.registration_date).days < 272 and
not mcct_status.is_booking and not mcct_status.is_stillbirth))
]
location_ids = get_location_hierarchy_by_id(
self.request_params.get("location_id", None),
self.domain,
self.request.couch_user,
CCT_only=True
)
q = _get_report_query(start_date, end_date, filtered_case_ids, location_ids)
if len(exclude_form_ids) > 0:
q["filter"]["and"].append({"not": {"ids": {"values": exclude_form_ids}}})
xmlnss = _get_relevant_xmlnss_for_service_type(self.request.GET.get("service_type_filter"))
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
modify_close = [_f for _f in ['Modify/Close Client'] if _f]
q["filter"]["and"].append({"not": {"terms": {"form.@name": modify_close}}})
q["sort"] = self.get_sorting_block() \
if self.get_sorting_block() else [{"form.meta.timeEnd" : {"order": "desc"}}]
if paginated:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms',
start_at=self.pagination.start, size=self.pagination.count)
else:
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_index='forms')
return self.es_response
def appointments_query(self, field_name, facets, check_response_state=False, base_query=None):
q = base_query or self.base_query
q["filter"] = {"and": [
{"term": {"workflow": WORKFLOW_REMINDER.lower()}},
{"term": {TC_STUB+field_name+'.case_type': 'cc_appointment'}},
]}
if check_response_state:
q["filter"]["and"].extend([{"term": {TC_STUB+field_name+'.response_state': 'confirmed'}}])
return es_query(q=q, es_url=ES_URLS['tc_sms'], facets=facets, size=0)
def get_call_center_domains():
q = {'fields': ['name']}
result = es_query(params={
'internal.using_call_center': True,
'is_active': True,
'is_snapshot': False
}, q=q)
hits = result.get('hits', {}).get('hits', {})
return [hit['fields']['name'] for hit in hits]
def query_es(self):
q = {
"query": {"query_string": {"query": self.query}},
"filter": {"and": ADD_TO_ES_FILTER["users"][:]},
"sort": {"username.exact": "asc"},
}
params = {"domain": self.domain, "is_active": not self.show_inactive}
self.es_results = es_query(
params=params, q=q, es_url=ES_URLS["users"], size=self.users_list_limit, start_at=self.users_list_skip
)
def get_call_center_domains():
q = {'fields': ['name']}
result = es_query(params={
'internal.using_call_center': True,
'is_active': True,
'is_snapshot': False
},
q=q)
hits = result.get('hits', {}).get('hits', {})
return [hit['fields']['name'] for hit in hits]
def selected_groups_query(self, request):
group_ids = self.selected_group_ids(request)
if not group_ids:
return []
q = {"query": {"filtered": {"filter": {"ids": {"values": group_ids}}}}}
return es_query(
es_url=ES_URLS["groups"],
q=q,
fields=['_id', 'name', "case_sharing", "reporting"],
)['hits']['hits']
def inactive_cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.now()
then = (now - timedelta(days=int(days))).strftime(DATE_FORMAT)
now = now.strftime(DATE_FORMAT)
q = {"query": {"bool": {"must_not": {"range": {"modified_on": {"from": then, "to": now}}}}}}
data = es_query(params={"domain.exact": domain, "closed": False}, q=q, es_url=CASE_INDEX + "/case/_search", size=1)
return data["hits"]["total"] if data.get("hits") else 0
def active_query(self, recipient_type):
q = self.base_query
q["query"]["bool"]["must"].append({"term": {"direction": "i"}})
q["query"]["bool"]["must"].append(
{"term": {
"couch_recipient_doc_type": recipient_type
}})
return es_query(q=q,
facets=['couch_recipient'],
es_url=ES_URLS['sms'],
size=0)
def total_distinct_users(domains=None):
"""
Get total number of users who've ever submitted a form.
"""
query = {"in": {"domain.exact": domains}} if domains is not None else {"match_all": {}}
q = {"query": query, "filter": {"and": ADD_TO_ES_FILTER["forms"][:]}}
res = es_query(q=q, facets=["form.meta.userID"], es_url=ES_URLS["forms"], size=0)
user_ids = reduce(list.__add__, [CouchUser.ids_by_domain(d) for d in domains], [])
terms = [t.get("term") for t in res["facets"]["form.meta.userID"]["terms"]]
return len(filter(lambda t: t and t not in WEIRD_USER_IDS and t in user_ids, terms))
def workflow_query(self, workflow=None, additional_facets=None):
additional_facets = additional_facets or []
q = self.base_query
if workflow:
q["filter"] = {"and": [{"term": {"workflow": workflow.lower()}}]}
else:
q["filter"] = {"and": [{"not": {"in": {"workflow": [w.lower() for w in WORKFLOWS]}}}]}
facets = ['couch_recipient_doc_type', 'direction'] + additional_facets
return es_query(q=q, facets=facets, es_url=ES_URLS['sms'], size=0)
def selected_groups_query(self, request):
group_ids = self.selected_group_ids(request)
if not group_ids:
return []
q = {"query": {"filtered": {"filter": {
"ids": {"values": group_ids}
}}}}
return es_query(
es_url=ES_URLS["groups"],
q=q,
fields=['_id', 'name', "case_sharing", "reporting"],
)['hits']['hits']
def selected_user_entries(self, request):
user_ids = self.selected_user_ids(request)
if not user_ids:
return []
q = {"query": {"filtered": {"filter": {
"ids": {"values": user_ids}
}}}}
res = es_query(
es_url=ES_URLS["users"],
q=q,
fields=['_id', 'username', 'first_name', 'last_name', 'doc_type'],
)
return [self.utils.user_tuple(hit['fields']) for hit in res['hits']['hits']]
def selected_user_entries(self, request):
user_ids = self.selected_user_ids(request)
if not user_ids:
return []
q = {"query": {"filtered": {"filter": {"ids": {"values": user_ids}}}}}
res = es_query(
es_url=ES_URLS["users"],
q=q,
fields=['_id', 'username', 'first_name', 'last_name', 'doc_type'],
)
return [
self.utils.user_tuple(hit['fields']) for hit in res['hits']['hits']
]
def get_rows(self, datespan):
block = self.block
q = {
"query": {
"filtered": {
"query": {"match_all": {}},
"filter": {
"bool": {
"must": [
{"term": {"domain.exact": self.domain}},
{"term": {"block_name.#value": block.lower()}},
]
}
},
}
}
}
es_filters = q["query"]["filtered"]["filter"]
if self.snapshot is None:
awcs = self.request.GET.getlist("awcs")
if awcs:
awcs_lower = [awc.lower() for awc in awcs]
awc_term = {
"or": [
{
"and": [
{"term": {"awc_name.#value": term}}
for term in re.split("\s", awc)
if not re.search(r"^\W+$", term) or re.search(r"^\w+", term)
]
}
for awc in awcs_lower
]
}
es_filters["bool"]["must"].append(awc_term)
gp = self.request_params.get("gp", None)
if gp:
users = CouchUser.by_domain(self.domain)
users_id = [
user._id
for user in users
if "user_data" in user and "gp" in user.user_data and user.user_data["gp"] == gp
]
es_filters["bool"]["must"].append({"terms": {"owner_id": users_id}})
is_open = self.request_params.get("is_open", None)
if is_open:
es_filters["bool"]["must"].append({"term": {"closed": is_open == "closed"}})
if self.default_case_type:
es_filters["bool"]["must"].append({"term": {"type.exact": self.default_case_type}})
logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q)))
return es_query(q=q, es_url=REPORT_CASE_INDEX + "/_search", dict_only=False)["hits"].get("hits", [])
def es_form_submissions(self, datespan=None, dict_only=False):
datespan = datespan or self.datespan
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"range": {
"form.meta.timeEnd": {
"from": datespan.startdate_param,
"to": datespan.enddate_param,
"include_upper": True}}}]}}}
q["filter"] = {"and": ADD_TO_ES_FILTER["forms"][:]}
facets = ['form.meta.userID']
return es_query(q=q, facets=facets, es_url=XFORM_INDEX + '/xform/_search', size=1, dict_only=dict_only)
def users_from_es(self):
q = {
"query": { "query_string": { "query": self.query }},
"filter": {"and": ADD_TO_ES_FILTER["users"][:]},
"sort": {'username.exact': 'asc'},
}
params = {
"domain": self.domain,
"is_active": not self.show_inactive,
}
results = es_query(params=params, q=q, es_url=ES_URLS["users"],
size=self.users_list_limit, start_at=self.users_list_skip)
users = [res['_source'] for res in results.get('hits', {}).get('hits', [])]
return [CommCareUser.wrap(user) for user in users]
def form_ids_response(self):
# returns a list of form_ids
# this is called using ReportDispatcher.dispatch(render_as='form_ids', ***) in
# the bulk_form_management_async task
from corehq.elastic import es_query
results = es_query(
params={'domain.exact': self.domain},
q=self.filters_as_es_query(),
es_index='forms',
fields=['_id'],
)
form_ids = [res['_id'] for res in results.get('hits', {}).get('hits', [])]
return form_ids
def es_total_cases(self, datespan=None, dict_only=False):
datespan = datespan or self.datespan
q = {"query": {
"bool": {
"must": [
{"match": {"domain.exact": self.domain}},
{"range": {"opened_on": {"lte": datespan.enddate_param}}}],
"must_not": {"range": {"closed_on": {"lt": datespan.startdate_param}}}}}}
if self.case_type:
q["query"]["bool"]["must"].append({"match": {"type.exact": self.case_type}})
facets = ['owner_id']
return es_query(q=q, facets=facets, es_url=CASE_INDEX + '/case/_search', size=1, dict_only=dict_only)
def cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.utcnow()
then = json_format_datetime(now - timedelta(days=int(days)))
now = json_format_datetime(now)
q = {"query": {
"range": {
"modified_on": {
"from": then,
"to": now}}}}
data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_url=CASE_INDEX + '/case/_search', size=1)
return data['hits']['total'] if data.get('hits') else 0
def form_ids_response(self):
# returns a list of form_ids
# this is called using ReportDispatcher.dispatch(render_as='form_ids', ***) in
# the bulk_form_management_async task
from corehq.elastic import es_query
results = es_query(
params={'domain.exact': self.domain},
q=self.filters_as_es_query(),
es_index='forms',
fields=['_id'],
)
form_ids = [
res['_id'] for res in results.get('hits', {}).get('hits', [])
]
return form_ids
def options(self):
q = {
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [
{
"term": {
"domain.exact": self.domain
}
},
{
"term": {
"type.exact": "participant"
}
},
]
}
}
}
}
es_filters = q["query"]["filtered"]["filter"]
def _filter_gen(key, flist):
return {"terms": {key: [item.lower() for item in flist if item]}}
user = self.request.couch_user
if not user.is_web_user():
owner_ids = user.get_group_ids()
user_ids = [user._id]
owner_filters = _filter_gen('owner_id', owner_ids)
user_filters = _filter_gen('user_id', user_ids)
filters = filter(None, [owner_filters, user_filters])
subterms = []
subterms.append({'or': filters})
es_filters["and"].append({'and': subterms} if subterms else {})
es_results = es_query(q=q,
es_url=REPORT_CASE_INDEX + '/_search',
dict_only=False)
return [(case['_source']['_id'],
case['_source']['full_name']['#value'])
for case in es_results['hits'].get('hits', [])]
def query_es(self):
is_simple, query = smart_query_string(self.query)
default_fields = ["username.exact", "last_name", "first_name"]
q = {
"query": {"query_string": {
"query": query,
"default_operator": "AND",
"fields": default_fields if is_simple else None
}},
"filter": {"and": ADD_TO_ES_FILTER["users"][:]},
"sort": {'username.exact': 'asc'},
}
params = {
"domain": self.domain,
"is_active": not self.show_inactive,
}
self.es_results = es_query(params=params, q=q, es_url=ES_URLS["users"],
size=self.users_list_limit, start_at=self.users_list_skip)
def get_rows(self, datespan):
block = self.block
q = {
"query": {
"filtered": {
"query": {
"match_all": {},
},
"filter": {
"bool": {
"must": [
{"term": {"domain.exact": self.domain}},
{"term": {"block_name.#value": block.lower()}}
]
}
}
}
}
}
es_filters = q["query"]["filtered"]["filter"]
if self.snapshot is None:
awcs = self.request.GET.getlist('awcs')
if awcs:
awcs_lower = [awc.lower() for awc in awcs]
awc_term = {
"or":
[{"and": [{"term": {"awc_name.#value": term}} for term in re.split('\s', awc)
if not re.search(r'^\W+$', term) or re.search(r'^\w+', term)]
} for awc in awcs_lower]
}
es_filters["bool"]["must"].append(awc_term)
gp = self.request_params.get('gp', None)
if gp:
users = CouchUser.by_domain(self.domain)
users_id = [user._id for user in users if 'user_data' in user and 'gp' in user.user_data and user.user_data['gp'] == gp]
es_filters["bool"]["must"].append({"terms": {"owner_id": users_id}})
is_open = self.request_params.get('is_open', None)
if is_open:
es_filters["bool"]["must"].append({"term": {"closed": is_open == 'closed'}})
if self.default_case_type:
es_filters["bool"]["must"].append({"term": {"type.exact": self.default_case_type}})
logging.info("ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(q)))
return es_query(q=q, es_url=REPORT_CASE_INDEX + '/_search', dict_only=False)['hits'].get('hits', [])
def query_es(self):
q = {
"filter": {
"and": ADD_TO_ES_FILTER["users"][:]
},
"sort": {
'username.exact': 'asc'
},
}
default_fields = ["username.exact", "last_name", "first_name"]
q["query"] = search_string_query(self.query, default_fields)
params = {
"domain": self.domain,
"is_active": not self.show_inactive,
}
self.es_results = es_query(params=params,
q=q,
es_url=ES_URLS["users"],
size=self.users_list_limit,
start_at=self.users_list_skip)
def workflow_query(self, workflow=None, additional_facets=None):
additional_facets = additional_facets or []
q = self.base_query
if workflow:
q["filter"] = {"and": [{"term": {"workflow": workflow.lower()}}]}
else:
q["filter"] = {
"and": [{
"not": {
"in": {
"workflow": [w.lower() for w in WORKFLOWS]
}
}
}]
}
facets = ['couch_recipient_doc_type', 'direction'] + additional_facets
return es_query(q=q, facets=facets, es_url=ES_URLS['sms'], size=0)
def es_results(self):
q = {
"query": {
"filtered": {
"query": {},
"filter": {
"bool": {
"must": [
{
"term": {
"domain.exact": self.domain
}
},
]
}
}
}
},
"size": self.pagination.count,
"from": self.pagination.start,
}
es_filters = q["query"]["filtered"]["filter"]
if self.awcs:
awc_term = get_nested_terms_filter("user_data.awc", self.awcs)
es_filters["bool"]["must"].append(awc_term)
elif self.gp:
gp_term = get_nested_terms_filter("user_data.gp", self.gp)
es_filters["bool"]["must"].append(gp_term)
elif self.blocks:
block_term = get_nested_terms_filter("user_data.block",
self.blocks)
es_filters["bool"]["must"].append(block_term)
q["query"]["filtered"]["query"].update({"match_all": {}})
logging.info(
"ESlog: [%s.%s] ESquery: %s" %
(self.__class__.__name__, self.domain, simplejson.dumps(q)))
return es_query(q=q,
es_url=USER_INDEX + '/_search',
dict_only=False,
start_at=self.pagination.start,
size=self.pagination.count)
def _query_es(limit, skip, query=None):
web_user_filter = [
{
"term": {
"user.domain_memberships.domain": domain
}
},
{
"term": {
"doc_type": "WebUser"
}
},
{
"term": {
"base_doc": "couchuser"
}
},
{
"term": {
"is_active": True
}
},
]
q = {
"filter": {
"and": web_user_filter
},
"sort": {
'username.exact': 'asc'
},
}
default_fields = ["username", "last_name", "first_name"]
q["query"] = search_string_query(query, default_fields)
return es_query(
params={},
q=q,
es_index='users',
size=limit,
start_at=skip,
)
def _get_filtered_cases(self, start_date, end_date):
query = {
"query": {
"bool": {
"must_not": [
{"range": {"modified_on.date": {"lt": start_date}}},
{"range": {"opened_on.date": {"gt": end_date}}}
]
}
}
}
case_search = self.request.GET.get("case_search", "")
if len(case_search) > 0:
query["filter"] = {
"and": [
{"regexp": {"name.exact": ".*?%s.*?" % case_search}}
]
}
es_response = es_query(params={"domain.exact": self.domain}, q=query, es_index='cases')
return [res['_source']['_id'] for res in es_response.get('hits', {}).get('hits', [])]
def es_results(self):
return es_query(
params={'domain.exact': self.domain},
q={
'query': {
'range': {
self.time_field: {
'from': self.datespan.startdate_param,
'to': self.datespan.enddate_param,
'include_upper': False,
}
}
},
'filter': {
'and': (ADD_TO_ES_FILTER['forms'] +
list(self._es_extra_filters()))
},
'sort': self.get_sorting_block(),
},
es_url=XFORM_INDEX + '/xform/_search',
start_at=self.pagination.start,
size=self.pagination.count,
)