def all_relevant_forms(self): selected_forms = FormsByApplicationFilter.get_value( self.request, self.domain) if self.request.GET.get('%s_unknown' % FormsByApplicationFilter.slug) == 'yes': return selected_forms # filter this result by submissions within this time frame key = make_form_couch_key(self.domain, by_submission_time=getattr( self, 'by_submission_time', True)) data = get_db().view( 'reports_forms/all_forms', reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ).all() all_submitted_forms = set([ FormsByApplicationFilter.make_xmlns_app_key( d['value']['xmlns'], d['value']['app_id']) for d in data ]) relevant_forms = all_submitted_forms.intersection( set(selected_forms.keys())) all_submitted_xmlns = [d['value']['xmlns'] for d in data] fuzzy_xmlns = set([ k for k in selected_forms.keys() if (FormsByApplicationFilter.fuzzy_slug in k and FormsByApplicationFilter.split_xmlns_app_key( k, only_xmlns=True) in all_submitted_xmlns) ]) relevant_forms = relevant_forms.union(fuzzy_xmlns) return dict([(k, selected_forms[k]) for k in relevant_forms])
def run_query(self, es_query): es_results = super(XFormES, self).run_query(es_query) #hack, walk the results again, and if we have xmlns, populate human readable names # Note that `get_unknown_form_name` does not require the request, which is also # not necessarily available here. So `None` is passed here. form_filter = FormsByApplicationFilter(None, domain=self.domain) for res in es_results.get('hits', {}).get('hits', []): if '_source' in res: res_source = restore_property_dict(res['_source']) res['_source'] = res_source xmlns = res['_source'].get('xmlns', None) name = None if xmlns: name = form_filter.get_unknown_form_name( xmlns, app_id=res['_source'].get('app_id', None), none_if_not_found=True) if not name: name = 'unknown' # try to fix it below but this will be the default # fall back try: if res['_source']['form'].get('@name', None): name = res['_source']['form']['@name'] else: backup = res['_source']['form'].get( '#type', 'data') if backup != 'data': name = backup except (TypeError, KeyError): pass res['_source']['es_readable_name'] = name return es_results
def all_relevant_forms(self): selected_forms = FormsByApplicationFilter.get_value(self.request, self.domain) if self.request.GET.get('%s_unknown' % FormsByApplicationFilter.slug) == 'yes': return selected_forms # filter this result by submissions within this time frame key = make_form_couch_key(self.domain, by_submission_time=getattr(self, 'by_submission_time', True)) data = get_db().view('reports_forms/all_forms', reduce=False, startkey=key+[self.datespan.startdate_param_utc], endkey=key+[self.datespan.enddate_param_utc], ).all() all_submitted_forms = set([FormsByApplicationFilter.make_xmlns_app_key(d['value']['xmlns'], d['value']['app_id']) for d in data]) relevant_forms = all_submitted_forms.intersection(set(selected_forms.keys())) all_submitted_xmlns = [d['value']['xmlns'] for d in data] fuzzy_xmlns = set([k for k in selected_forms.keys() if (FormsByApplicationFilter.fuzzy_slug in k and FormsByApplicationFilter.split_xmlns_app_key(k, only_xmlns=True) in all_submitted_xmlns)]) relevant_forms = relevant_forms.union(fuzzy_xmlns) return dict([(k, selected_forms[k]) for k in relevant_forms])
def run_query(self, es_query): es_results = super(XFormES, self).run_query(es_query) #hack, walk the results again, and if we have xmlns, populate human readable names # Note that `get_unknown_form_name` does not require the request, which is also # not necessarily available here. So `None` is passed here. form_filter = FormsByApplicationFilter(None, domain=self.domain) for res in es_results.get('hits', {}).get('hits', []): if '_source' in res: res_source = restore_property_dict(res['_source']) res['_source'] = res_source xmlns = res['_source'].get('xmlns', None) name = None if xmlns: name = form_filter.get_unknown_form_name(xmlns, app_id=res['_source'].get('app_id', None), none_if_not_found=True) if not name: name = 'unknown' # try to fix it below but this will be the default # fall back try: if res['_source']['form'].get('@name', None): name = res['_source']['form']['@name'] else: backup = res['_source']['form'].get('#type', 'data') if backup != 'data': name = backup except (TypeError, KeyError): pass res['_source']['es_readable_name'] = name return es_results
def run_query(self, es_query, **kwargs): es_results = super(XFormES, self).run_query(es_query) # hack, walk the results again, and if we have xmlns, populate human readable names # Note that `get_unknown_form_name` does not require the request, which is also # not necessarily available here. So `None` is passed here. form_filter = FormsByApplicationFilter(None, domain=self.domain) for res in es_results.get("hits", {}).get("hits", []): if "_source" in res: xmlns = res["_source"].get("xmlns", None) name = None if xmlns: name = form_filter.get_unknown_form_name( xmlns, app_id=res["_source"].get("app_id", None), none_if_not_found=True ) if not name: name = "unknown" # try to fix it below but this will be the default # fall back try: if res["_source"]["form"].get("@name", None): name = res["_source"]["form"]["@name"] else: backup = res["_source"]["form"].get("#type", "data") if backup != "data": name = backup except (TypeError, KeyError): pass res["_source"]["es_readable_name"] = name return es_results
def test_get_filtered_data_by_app_id_missing(self): params = FormsByApplicationFilterParams([ _make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE), _make_filter(PARAM_SLUG_APP_ID, 'missing') ]) results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(self.domain, params) self.assertEqual(0, len(results))
def es_query(self): time_filter = form_es.submitted if self.by_submission_time else form_es.completed mobile_user_and_group_slugs = self.request.GET.getlist(EMWF.slug) query = (form_es.FormES().domain(self.domain).filter( time_filter( gte=self.datespan.startdate, lt=self.datespan.enddate_adjusted)).filter( self._get_users_filter(mobile_user_and_group_slugs))) if not self.request.can_access_all_locations: query = query.filter(self.scope_filter()) # filter results by app and xmlns if applicable if FormsByApplicationFilter.has_selections(self.request): form_values = self.all_relevant_forms.values() if form_values: query = query.OR(*[self._form_filter(f) for f in form_values]) # Exclude system forms unless they selected "Unknown User" if HQUserType.UNKNOWN not in EMWF.selected_user_types( mobile_user_and_group_slugs): query = query.NOT(form_es.xmlns(SYSTEM_FORM_XMLNS)) return query
def test_get_filtered_data_by_app_id_missing(self): params = FormsByApplicationFilterParams([ _make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE), _make_filter(PARAM_SLUG_APP_ID, 'missing') ]) results = FormsByApplicationFilter.get_filtered_data_for_parsed_params( self.domain, params) self.assertEqual(0, len(results))
def test_get_filtered_data_by_app_id(self): params = FormsByApplicationFilterParams([ _make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE), _make_filter(PARAM_SLUG_APP_ID, self.app.id) ]) results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(self.domain, params) self.assertEqual(2, len(results)) for i, details in enumerate(results): self._assert_form_details_match(i, details)
def test_get_filtered_data_by_app_id(self): params = FormsByApplicationFilterParams([ _make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE), _make_filter(PARAM_SLUG_APP_ID, self.app.id) ]) results = FormsByApplicationFilter.get_filtered_data_for_parsed_params( self.domain, params) self.assertEqual(2, len(results)) for i, details in enumerate(results): self._assert_form_details_match(i, details)
def _es_extra_filters(self): if FormsByApplicationFilter.has_selections(self.request): def form_filter(form): app_id = form.get('app_id', None) if app_id and app_id != MISSING_APP_ID: return {'and': [{'term': {'xmlns.exact': form['xmlns']}}, {'term': {'app_id': app_id}}]} return {'term': {'xmlns.exact': form['xmlns']}} form_values = self.all_relevant_forms.values() if form_values: yield {'or': [form_filter(f) for f in form_values]} truthy_only = functools.partial(filter, None) mobile_user_and_group_slugs = self.request.GET.getlist(ExpandedMobileWorkerFilter.slug) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, mobile_user_and_group_slugs, include_inactive=True ) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only( u.user_id for u in users_data.combined_users ) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only( user.user_id for user in negated_ids ) } } } props = truthy_only(self.request.GET.get('form_data', '').split(',')) for prop in props: yield { 'term': {'__props_for_querying': prop} } if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types(mobile_user_and_group_slugs): yield { 'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}} }
def test_get_filtered_data_by_module_id(self): for i in range(2): params = FormsByApplicationFilterParams([ _make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE), _make_filter(PARAM_SLUG_APP_ID, self.app.id), _make_filter(PARAM_SLUG_MODULE, str(i)), ]) results = FormsByApplicationFilter.get_filtered_data_for_parsed_params(self.domain, params) self.assertEqual(1, len(results)) details = results[0] self._assert_form_details_match(i, details)
def _es_extra_filters(self): if FormsByApplicationFilter.has_selections(self.request): def form_filter(form): app_id = form.get('app_id', None) if app_id and app_id != MISSING_APP_ID: return { 'and': [{ 'term': { 'xmlns.exact': form['xmlns'] } }, { 'term': { 'app_id': app_id } }] } return {'term': {'xmlns.exact': form['xmlns']}} form_values = self.all_relevant_forms.values() if form_values: yield {'or': [form_filter(f) for f in form_values]} truthy_only = functools.partial(filter, None) mobile_user_and_group_slugs = self.request.GET.getlist( ExpandedMobileWorkerFilter.slug) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, mobile_user_and_group_slugs, include_inactive=True) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only(u.user_id for u in users_data.combined_users) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only(user.user_id for user in negated_ids) } } } if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types( mobile_user_and_group_slugs): yield {'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}}}
def test_get_filtered_data_by_module_id(self): for i in range(2): params = FormsByApplicationFilterParams([ _make_filter(PARAM_SLUG_STATUS, PARAM_VALUE_STATUS_ACTIVE), _make_filter(PARAM_SLUG_APP_ID, self.app.id), _make_filter(PARAM_SLUG_MODULE, str(i)), ]) results = FormsByApplicationFilter.get_filtered_data_for_parsed_params( self.domain, params) self.assertEqual(1, len(results)) details = results[0] self._assert_form_details_match(i, details)
def _es_extra_filters(self): if FormsByApplicationFilter.has_selections(self.request): def form_filter(form): app_id = form.get('app_id', None) if app_id and app_id != MISSING_APP_ID: return { 'and': [{ 'term': { 'xmlns.exact': form['xmlns'] } }, { 'term': { 'app_id': app_id } }] } return {'term': {'xmlns.exact': form['xmlns']}} form_values = self.all_relevant_forms.values() if form_values: yield {'or': [form_filter(f) for f in form_values]} truthy_only = functools.partial(filter, None) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, self.request, True, True, include_inactive=True) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only(u.user_id for u in users_data.combined_users) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only(user.user_id for user in negated_ids) } } } props = truthy_only(self.request.GET.get('form_data', '').split(',')) for prop in props: yield {'term': {'__props_for_querying': prop.lower()}}
def es_query(self): time_filter = form_es.submitted if self.by_submission_time else form_es.completed mobile_user_and_group_slugs = self.request.GET.getlist(EMWF.slug) query = (form_es.FormES() .domain(self.domain) .filter(time_filter(gte=self.datespan.startdate, lt=self.datespan.enddate_adjusted)) .filter(self._get_users_filter(mobile_user_and_group_slugs))) # filter results by app and xmlns if applicable if FormsByApplicationFilter.has_selections(self.request): form_values = self.all_relevant_forms.values() if form_values: query = query.OR(*[self._form_filter(f) for f in form_values]) # Exclude system forms unless they selected "Unknown User" if HQUserType.UNKNOWN not in EMWF.selected_user_types(mobile_user_and_group_slugs): query = query.NOT(form_es.xmlns(SYSTEM_FORM_XMLNS)) return query
def all_relevant_forms(self): return FormsByApplicationFilter.get_value(self.request, self.domain)