def all_relevant_forms(self):
selected_forms = FormsByApplicationFilter.get_value(
self.request, self.domain)
if self.request.GET.get('%s_unknown' %
FormsByApplicationFilter.slug) == 'yes':
return selected_forms
# filter this result by submissions within this time frame
key = make_form_couch_key(self.domain,
by_submission_time=getattr(
self, 'by_submission_time', True))
data = get_db().view(
'reports_forms/all_forms',
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
).all()
all_submitted_forms = set([
FormsByApplicationFilter.make_xmlns_app_key(
d['value']['xmlns'], d['value']['app_id']) for d in data
])
relevant_forms = all_submitted_forms.intersection(
set(selected_forms.keys()))
all_submitted_xmlns = [d['value']['xmlns'] for d in data]
fuzzy_xmlns = set([
k for k in selected_forms.keys()
if (FormsByApplicationFilter.fuzzy_slug in k
and FormsByApplicationFilter.split_xmlns_app_key(
k, only_xmlns=True) in all_submitted_xmlns)
])
relevant_forms = relevant_forms.union(fuzzy_xmlns)
return dict([(k, selected_forms[k]) for k in relevant_forms])
def run_query(self, es_query):
es_results = super(XFormES, self).run_query(es_query)
#hack, walk the results again, and if we have xmlns, populate human readable names
# Note that `get_unknown_form_name` does not require the request, which is also
# not necessarily available here. So `None` is passed here.
form_filter = FormsByApplicationFilter(None, domain=self.domain)
for res in es_results.get('hits', {}).get('hits', []):
if '_source' in res:
res_source = restore_property_dict(res['_source'])
res['_source'] = res_source
xmlns = res['_source'].get('xmlns', None)
name = None
if xmlns:
name = form_filter.get_unknown_form_name(
xmlns,
app_id=res['_source'].get('app_id', None),
none_if_not_found=True)
if not name:
name = 'unknown' # try to fix it below but this will be the default
# fall back
try:
if res['_source']['form'].get('@name', None):
name = res['_source']['form']['@name']
else:
backup = res['_source']['form'].get(
'#type', 'data')
if backup != 'data':
name = backup
except (TypeError, KeyError):
pass
res['_source']['es_readable_name'] = name
return es_results
def all_relevant_forms(self):
selected_forms = FormsByApplicationFilter.get_value(self.request, self.domain)
if self.request.GET.get('%s_unknown' % FormsByApplicationFilter.slug) == 'yes':
return selected_forms
# filter this result by submissions within this time frame
key = make_form_couch_key(self.domain, by_submission_time=getattr(self, 'by_submission_time', True))
data = get_db().view('reports_forms/all_forms',
reduce=False,
startkey=key+[self.datespan.startdate_param_utc],
endkey=key+[self.datespan.enddate_param_utc],
).all()
all_submitted_forms = set([FormsByApplicationFilter.make_xmlns_app_key(d['value']['xmlns'], d['value']['app_id'])
for d in data])
relevant_forms = all_submitted_forms.intersection(set(selected_forms.keys()))
all_submitted_xmlns = [d['value']['xmlns'] for d in data]
fuzzy_xmlns = set([k for k in selected_forms.keys()
if (FormsByApplicationFilter.fuzzy_slug in k and
FormsByApplicationFilter.split_xmlns_app_key(k, only_xmlns=True) in all_submitted_xmlns)])
relevant_forms = relevant_forms.union(fuzzy_xmlns)
return dict([(k, selected_forms[k]) for k in relevant_forms])
def run_query(self, es_query):
es_results = super(XFormES, self).run_query(es_query)
#hack, walk the results again, and if we have xmlns, populate human readable names
# Note that `get_unknown_form_name` does not require the request, which is also
# not necessarily available here. So `None` is passed here.
form_filter = FormsByApplicationFilter(None, domain=self.domain)
for res in es_results.get('hits', {}).get('hits', []):
if '_source' in res:
res_source = restore_property_dict(res['_source'])
res['_source'] = res_source
xmlns = res['_source'].get('xmlns', None)
name = None
if xmlns:
name = form_filter.get_unknown_form_name(xmlns,
app_id=res['_source'].get('app_id',
None),
none_if_not_found=True)
if not name:
name = 'unknown' # try to fix it below but this will be the default
# fall back
try:
if res['_source']['form'].get('@name', None):
name = res['_source']['form']['@name']
else:
backup = res['_source']['form'].get('#type', 'data')
if backup != 'data':
name = backup
except (TypeError, KeyError):
pass
res['_source']['es_readable_name'] = name
return es_results
def run_query(self, es_query, **kwargs):
es_results = super(XFormES, self).run_query(es_query)
# hack, walk the results again, and if we have xmlns, populate human readable names
# Note that `get_unknown_form_name` does not require the request, which is also
# not necessarily available here. So `None` is passed here.
form_filter = FormsByApplicationFilter(None, domain=self.domain)
for res in es_results.get("hits", {}).get("hits", []):
if "_source" in res:
xmlns = res["_source"].get("xmlns", None)
name = None
if xmlns:
name = form_filter.get_unknown_form_name(
xmlns, app_id=res["_source"].get("app_id", None), none_if_not_found=True
)
if not name:
name = "unknown" # try to fix it below but this will be the default
# fall back
try:
if res["_source"]["form"].get("@name", None):
name = res["_source"]["form"]["@name"]
else:
backup = res["_source"]["form"].get("#type", "data")
if backup != "data":
name = backup
except (TypeError, KeyError):
pass
res["_source"]["es_readable_name"] = name
return es_results
def test_get_filtered_data_by_app_id_missing(self):
params = FormsByApplicationFilterParams([
_make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE),
_make_filter(PARAM_SLUG_APP_ID, 'missing')
])
results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(self.domain, params)
self.assertEqual(0, len(results))
def es_query(self):
time_filter = form_es.submitted if self.by_submission_time else form_es.completed
mobile_user_and_group_slugs = self.request.GET.getlist(EMWF.slug)
query = (form_es.FormES().domain(self.domain).filter(
time_filter(
gte=self.datespan.startdate,
lt=self.datespan.enddate_adjusted)).filter(
self._get_users_filter(mobile_user_and_group_slugs)))
if not self.request.can_access_all_locations:
query = query.filter(self.scope_filter())
# filter results by app and xmlns if applicable
if FormsByApplicationFilter.has_selections(self.request):
form_values = self.all_relevant_forms.values()
if form_values:
query = query.OR(*[self._form_filter(f) for f in form_values])
# Exclude system forms unless they selected "Unknown User"
if HQUserType.UNKNOWN not in EMWF.selected_user_types(
mobile_user_and_group_slugs):
query = query.NOT(form_es.xmlns(SYSTEM_FORM_XMLNS))
return query
def test_get_filtered_data_by_app_id_missing(self):
params = FormsByApplicationFilterParams([
_make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE),
_make_filter(PARAM_SLUG_APP_ID, 'missing')
])
results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(
self.domain, params)
self.assertEqual(0, len(results))
def test_get_filtered_data_by_app_id(self):
params = FormsByApplicationFilterParams([
_make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE),
_make_filter(PARAM_SLUG_APP_ID, self.app.id)
])
results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(self.domain, params)
self.assertEqual(2, len(results))
for i, details in enumerate(results):
self._assert_form_details_match(i, details)
def test_get_filtered_data_by_app_id(self):
params = FormsByApplicationFilterParams([
_make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE),
_make_filter(PARAM_SLUG_APP_ID, self.app.id)
])
results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(
self.domain, params)
self.assertEqual(2, len(results))
for i, details in enumerate(results):
self._assert_form_details_match(i, details)
def _es_extra_filters(self):
if FormsByApplicationFilter.has_selections(self.request):
def form_filter(form):
app_id = form.get('app_id', None)
if app_id and app_id != MISSING_APP_ID:
return {'and': [{'term': {'xmlns.exact': form['xmlns']}},
{'term': {'app_id': app_id}}]}
return {'term': {'xmlns.exact': form['xmlns']}}
form_values = self.all_relevant_forms.values()
if form_values:
yield {'or': [form_filter(f) for f in form_values]}
truthy_only = functools.partial(filter, None)
mobile_user_and_group_slugs = self.request.GET.getlist(ExpandedMobileWorkerFilter.slug)
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain,
mobile_user_and_group_slugs,
include_inactive=True
)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID': truthy_only(
u.user_id for u in users_data.combined_users
)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID': truthy_only(
user.user_id for user in negated_ids
)
}
}
}
props = truthy_only(self.request.GET.get('form_data', '').split(','))
for prop in props:
yield {
'term': {'__props_for_querying': prop}
}
if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types(mobile_user_and_group_slugs):
yield {
'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}}
}
def test_get_filtered_data_by_module_id(self):
for i in range(2):
params = FormsByApplicationFilterParams([
_make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE),
_make_filter(PARAM_SLUG_APP_ID, self.app.id),
_make_filter(PARAM_SLUG_MODULE, str(i)),
])
results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(self.domain, params)
self.assertEqual(1, len(results))
details = results[0]
self._assert_form_details_match(i, details)
def _es_extra_filters(self):
if FormsByApplicationFilter.has_selections(self.request):
def form_filter(form):
app_id = form.get('app_id', None)
if app_id and app_id != MISSING_APP_ID:
return {
'and': [{
'term': {
'xmlns.exact': form['xmlns']
}
}, {
'term': {
'app_id': app_id
}
}]
}
return {'term': {'xmlns.exact': form['xmlns']}}
form_values = self.all_relevant_forms.values()
if form_values:
yield {'or': [form_filter(f) for f in form_values]}
truthy_only = functools.partial(filter, None)
mobile_user_and_group_slugs = self.request.GET.getlist(
ExpandedMobileWorkerFilter.slug)
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain, mobile_user_and_group_slugs, include_inactive=True)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID':
truthy_only(u.user_id for u in users_data.combined_users)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID':
truthy_only(user.user_id for user in negated_ids)
}
}
}
if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types(
mobile_user_and_group_slugs):
yield {'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}}}
def test_get_filtered_data_by_module_id(self):
for i in range(2):
params = FormsByApplicationFilterParams([
_make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE),
_make_filter(PARAM_SLUG_APP_ID, self.app.id),
_make_filter(PARAM_SLUG_MODULE, str(i)),
])
results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(
self.domain, params)
self.assertEqual(1, len(results))
details = results[0]
self._assert_form_details_match(i, details)
def _es_extra_filters(self):
if FormsByApplicationFilter.has_selections(self.request):
def form_filter(form):
app_id = form.get('app_id', None)
if app_id and app_id != MISSING_APP_ID:
return {
'and': [{
'term': {
'xmlns.exact': form['xmlns']
}
}, {
'term': {
'app_id': app_id
}
}]
}
return {'term': {'xmlns.exact': form['xmlns']}}
form_values = self.all_relevant_forms.values()
if form_values:
yield {'or': [form_filter(f) for f in form_values]}
truthy_only = functools.partial(filter, None)
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain, self.request, True, True, include_inactive=True)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID':
truthy_only(u.user_id for u in users_data.combined_users)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID':
truthy_only(user.user_id for user in negated_ids)
}
}
}
props = truthy_only(self.request.GET.get('form_data', '').split(','))
for prop in props:
yield {'term': {'__props_for_querying': prop.lower()}}
def es_query(self):
time_filter = form_es.submitted if self.by_submission_time else form_es.completed
mobile_user_and_group_slugs = self.request.GET.getlist(EMWF.slug)
query = (form_es.FormES()
.domain(self.domain)
.filter(time_filter(gte=self.datespan.startdate,
lt=self.datespan.enddate_adjusted))
.filter(self._get_users_filter(mobile_user_and_group_slugs)))
# filter results by app and xmlns if applicable
if FormsByApplicationFilter.has_selections(self.request):
form_values = self.all_relevant_forms.values()
if form_values:
query = query.OR(*[self._form_filter(f) for f in form_values])
# Exclude system forms unless they selected "Unknown User"
if HQUserType.UNKNOWN not in EMWF.selected_user_types(mobile_user_and_group_slugs):
query = query.NOT(form_es.xmlns(SYSTEM_FORM_XMLNS))
return query
def all_relevant_forms(self):
return FormsByApplicationFilter.get_value(self.request, self.domain)
