def generate_case_export_payload(domain, include_closed, format, group, user_filter, process=None): """ Returns a FileWrapper object, which only the file backend in django-soil supports """ status = "all" if include_closed else "open" case_ids = CommCareCase.get_all_cases(domain, status=status, wrapper=lambda r: r["id"]) class stream_cases(object): def __init__(self, all_case_ids): self.all_case_ids = all_case_ids def __iter__(self): for case_ids in chunked(self.all_case_ids, 500): for case in wrapped_docs(CommCareCase, case_ids): yield case def __len__(self): return len(self.all_case_ids) # todo deal with cached user dict here users = get_all_users_by_domain(domain, group=group, user_filter=user_filter) groups = Group.get_case_sharing_groups(domain) fd, path = tempfile.mkstemp() with os.fdopen(fd, "wb") as file: workbook = WorkBook(file, format) export_cases_and_referrals( domain, stream_cases(case_ids), workbook, users=users, groups=groups, process=process ) export_users(users, workbook) workbook.close() return FileWrapper(open(path))
def export_all_form_metadata_async(req, domain): datespan = req.datespan if req.GET.get("startdate") and req.GET.get("enddate") else None group_id = req.GET.get("group") ufilter = UserTypeFilter.get_user_filter(req)[0] users = util.get_all_users_by_domain( domain=domain, group=group_id, user_filter=ufilter, simplified=True, include_inactive=True ) user_ids = filter(None, [u["user_id"] for u in users]) format = req.GET.get("format", Format.XLS_2007) filename = "%s_forms" % domain download = DownloadBase() download.set_task(create_metadata_export.delay( download.download_id, domain, format=format, filename=filename, datespan=datespan, user_ids=user_ids, )) return download.get_start_response()
def export_all_form_metadata_async(req, domain): datespan = req.datespan if req.GET.get("startdate") and req.GET.get( "enddate") else None group_id = req.GET.get("group") ufilter = UserTypeFilter.get_user_filter(req)[0] users = list( util.get_all_users_by_domain(domain=domain, group=group_id, user_filter=ufilter, simplified=True)) user_ids = filter(None, [u["user_id"] for u in users]) format = req.GET.get("format", Format.XLS_2007) filename = "%s_forms" % domain download = DownloadBase() download.set_task( create_metadata_export.delay( download.download_id, domain, format=format, filename=filename, datespan=datespan, user_ids=user_ids, )) return download.get_start_response()
def es_query(self): if not getattr(self, 'es_response', None): q = { "query": { "range": { self.time_field: { "from": self.datespan.startdate_param, "to": self.datespan.enddate_param, "include_upper": False}}}, "filter": {"and": ADD_TO_ES_FILTER["forms"][:]}} xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()]) if xmlnss: q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}}) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(self.domain, self.request, True, True) if "t__0" not in self.request.GET.getlist("emw") or users_data["admin_and_demo_users"]: q["filter"]["and"].append( {"terms": {"form.meta.userID": filter(None, [u["user_id"] for u in users_data["combined_users"]])}}) else: negated_ids = util.get_all_users_by_domain(self.domain, user_filter=HQUserType.all_but_users(), simplified=True) ids = filter(None, [user['user_id'] for user in negated_ids]) q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}}) for cp in filter(None, self.request.GET.get('form_data', "").split(",")): q["filter"]["and"].append({"term": {"__props_for_querying": cp.lower()}}) q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{self.time_field : {"order": "desc"}}] self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search', start_at=self.pagination.start, size=self.pagination.count) return self.es_response
def users(self): """ todo: cache this baby """ if self._users is None: self._users = util.get_all_users_by_domain(self.domain, group=self.group_name, individual=self.individual, user_filter=self.user_filter, simplified=True) return self._users
def user_cacher(domain, cache_key, current_cache=None, refresh_stale=1800, cache_timeout=3600, **kwargs): from corehq.apps.reports.util import get_all_users_by_domain data = get_all_users_by_domain(domain, **kwargs) _cache_data(data, cache_key, current_cache=current_cache, refresh_stale=refresh_stale, cache_timeout=cache_timeout)
def get_all_users_by_domain(self, group=None, user_ids=None, user_filter=None, simplified=False): return list(util.get_all_users_by_domain( domain=self.domain, group=group, user_ids=user_ids, user_filter=user_filter, simplified=False, # override simplified to False CommCareUser=self.CommCareUser ))
def get_all_users_by_domain(self, group=None, user_ids=None, user_filter=None, simplified=False): return list(util.get_all_users_by_domain( domain=self.domain, group=group, user_ids=user_ids, user_filter=user_filter, simplified=simplified, CommCareUser=self.CommCareUser ))
def get_all_users_by_domain(self, group=None, individual=None, user_filter=None, simplified=False): return list(util.get_all_users_by_domain( domain=self.domain, group=group, individual=individual, user_filter=user_filter, simplified=simplified, CommCareUser=self.CommCareUser ))
def report_context(self): context = super(CaseReassignmentInterface, self).report_context active_users = util.get_all_users_by_domain(self.domain, user_filter=HQUserType.use_defaults(), simplified=True) context.update( users=[dict(ownerid=user.get('user_id'), name=user.get('username_in_report'), type="user") for user in active_users], groups=[dict(ownerid=group.get_id, name=group.name, type="group") for group in self.all_case_sharing_groups] ) return context
def users(self): group = self.group if not self.user_id else None user_ids = (self.user_id,) users = list( util.get_all_users_by_domain( domain=self.domain, user_ids=user_ids, group=group, simplified=True, CommCareUser=CommCareUserMemoizer() ) ) return sorted(users, key=itemgetter("raw_username"))
def _es_extra_filters(self): if FormsByApplicationFilter.has_selections(self.request): def form_filter(form): app_id = form.get('app_id', None) if app_id and app_id != MISSING_APP_ID: return {'and': [{'term': {'xmlns.exact': form['xmlns']}}, {'term': {'app_id': app_id}}]} return {'term': {'xmlns.exact': form['xmlns']}} form_values = self.all_relevant_forms.values() if form_values: yield {'or': [form_filter(f) for f in form_values]} truthy_only = functools.partial(filter, None) mobile_user_and_group_slugs = self.request.GET.getlist(ExpandedMobileWorkerFilter.slug) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, mobile_user_and_group_slugs, include_inactive=True ) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only( u.user_id for u in users_data.combined_users ) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only( user.user_id for user in negated_ids ) } } } props = truthy_only(self.request.GET.get('form_data', '').split(',')) for prop in props: yield { 'term': {'__props_for_querying': prop} } if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types(mobile_user_and_group_slugs): yield { 'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}} }
def users(self): group = self.group if not self.user_id else None user_ids = (self.user_id, ) users = list( util.get_all_users_by_domain(domain=self.domain, user_ids=user_ids, group=group, simplified=True, CommCareUser=CommCareUserMemoizer())) return sorted(users, key=itemgetter('raw_username'))
def _es_extra_filters(self): if FormsByApplicationFilter.has_selections(self.request): def form_filter(form): app_id = form.get('app_id', None) if app_id and app_id != MISSING_APP_ID: return { 'and': [{ 'term': { 'xmlns.exact': form['xmlns'] } }, { 'term': { 'app_id': app_id } }] } return {'term': {'xmlns.exact': form['xmlns']}} form_values = self.all_relevant_forms.values() if form_values: yield {'or': [form_filter(f) for f in form_values]} truthy_only = functools.partial(filter, None) mobile_user_and_group_slugs = self.request.GET.getlist( ExpandedMobileWorkerFilter.slug) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, mobile_user_and_group_slugs, include_inactive=True) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only(u.user_id for u in users_data.combined_users) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only(user.user_id for user in negated_ids) } } } if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types( mobile_user_and_group_slugs): yield {'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}}}
def export_data(req, domain): """ Download all data for a couchdbkit model """ try: export_tag = json.loads(req.GET.get("export_tag", "null") or "null") except ValueError: return HttpResponseBadRequest() group, users = util.get_group_params(domain, **json_request(req.GET)) include_errors = string_to_boolean(req.GET.get("include_errors", False)) kwargs = {"format": req.GET.get("format", Format.XLS_2007), "previous_export_id": req.GET.get("previous_export", None), "filename": export_tag, "use_cache": string_to_boolean(req.GET.get("use_cache", "True")), "max_column_size": int(req.GET.get("max_column_size", 2000)), "separator": req.GET.get("separator", "|")} user_filter, _ = FilterUsersField.get_user_filter(req) if user_filter: users_matching_filter = map(lambda x: x.get('user_id'), get_all_users_by_domain(domain, user_filter=user_filter, simplified=True)) def _ufilter(user): try: return user['form']['meta']['userID'] in users_matching_filter except KeyError: return False filter = _ufilter else: filter = SerializableFunction(util.group_filter, group=group) errors_filter = instances if not include_errors else None kwargs['filter'] = couchexport.util.intersect_functions(filter, errors_filter) if kwargs['format'] == 'raw': resp = export_raw_data([domain, export_tag], filename=export_tag) else: try: resp = export_data_shared([domain,export_tag], **kwargs) except UnsupportedExportFormat as e: return HttpResponseBadRequest(e) if resp: return resp else: messages.error(req, "Sorry, there was no data found for the tag '%s'." % export_tag) next = req.GET.get("next", "") if not next: next = export.ExcelExportReport.get_url(domain=domain) return HttpResponseRedirect(next)
def _es_extra_filters(self): if FormsByApplicationFilter.has_selections(self.request): def form_filter(form): app_id = form.get('app_id', None) if app_id and app_id != MISSING_APP_ID: return { 'and': [{ 'term': { 'xmlns.exact': form['xmlns'] } }, { 'term': { 'app_id': app_id } }] } return {'term': {'xmlns.exact': form['xmlns']}} form_values = self.all_relevant_forms.values() if form_values: yield {'or': [form_filter(f) for f in form_values]} truthy_only = functools.partial(filter, None) users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, self.request, True, True, include_inactive=True) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only(u.user_id for u in users_data.combined_users) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only(user.user_id for user in negated_ids) } } } props = truthy_only(self.request.GET.get('form_data', '').split(',')) for prop in props: yield {'term': {'__props_for_querying': prop.lower()}}
def export_data(req, domain): """ Download all data for a couchdbkit model """ try: export_tag = json.loads(req.GET.get("export_tag", "null") or "null") except ValueError: return HttpResponseBadRequest() group, users = util.get_group_params(domain, **json_request(req.GET)) include_errors = string_to_boolean(req.GET.get("include_errors", False)) kwargs = {"format": req.GET.get("format", Format.XLS_2007), "previous_export_id": req.GET.get("previous_export", None), "filename": export_tag, "use_cache": string_to_boolean(req.GET.get("use_cache", "True")), "max_column_size": int(req.GET.get("max_column_size", 2000)), "separator": req.GET.get("separator", "|")} user_filter, _ = UserTypeFilter.get_user_filter(req) if user_filter: users_matching_filter = map(lambda x: x.get('user_id'), get_all_users_by_domain(domain, user_filter=user_filter, simplified=True)) def _ufilter(user): try: return user['form']['meta']['userID'] in users_matching_filter except KeyError: return False filter = _ufilter else: filter = SerializableFunction(util.group_filter, group=group) errors_filter = instances if not include_errors else None kwargs['filter'] = couchexport.util.intersect_functions(filter, errors_filter) if kwargs['format'] == 'raw': resp = export_raw_data([domain, export_tag], filename=export_tag) else: try: resp = export_data_shared([domain, export_tag], **kwargs) except CouchExportException as e: return HttpResponseBadRequest(e) if resp: return resp else: messages.error(req, "Sorry, there was no data found for the tag '%s'." % export_tag) next = req.GET.get("next", "") if not next: next = export.ExcelExportReport.get_url(domain=domain) return HttpResponseRedirect(next)
def pull_users_and_groups(cls, domain, request, simplified_users=False, combined=False, CommCareUser=CommCareUser): emws = request.GET.getlist(cls.slug) users = [] user_ids = [u[3:] for u in filter(lambda s: s.startswith("u__"), emws)] if user_ids or "t__0" in emws: users = util.get_all_users_by_domain(domain=domain, user_ids=user_ids, simplified=simplified_users, CommCareUser=CommCareUser) user_type_ids = [int(t[3:]) for t in filter(lambda s: s.startswith("t__"), emws)] user_filter = tuple([HQUserToggle(id, id in user_type_ids) for id in range(4)]) other_users = util.get_all_users_by_domain(domain=domain, user_filter=user_filter, simplified=simplified_users, CommCareUser=CommCareUser) group_ids = [g[3:] for g in filter(lambda s: s.startswith("g__"), emws)] groups = [Group.get(g) for g in group_ids] ret = { "users": users + other_users, "admin_and_demo_users": other_users, "groups": groups, } if combined: user_dict = {} for group in groups: user_dict["%s|%s" % (group.name, group._id)] = util.get_all_users_by_domain( group=group, simplified=simplified_users ) users_in_groups = [user for sublist in user_dict.values() for user in sublist] ret["users_by_group"] = user_dict ret["combined_users"] = remove_dups(ret["users"] + users_in_groups, "user_id") return ret
def generate_case_export_payload(domain, include_closed, format, group, user_filter, process=None): """ Returns a FileWrapper object, which only the file backend in django-soil supports """ status = 'all' if include_closed else 'open' case_ids = CommCareCase.get_all_cases(domain, status=status, wrapper=lambda r: r['id']) class stream_cases(object): def __init__(self, all_case_ids): self.all_case_ids = all_case_ids def __iter__(self): for case_ids in chunked(self.all_case_ids, 500): for case in wrapped_docs(CommCareCase, case_ids): yield case def __len__(self): return len(self.all_case_ids) # todo deal with cached user dict here users = get_all_users_by_domain(domain, group=group, user_filter=user_filter) groups = Group.get_case_sharing_groups(domain) fd, path = tempfile.mkstemp() with os.fdopen(fd, 'wb') as file: workbook = WorkBook(file, format) export_cases_and_referrals(domain, stream_cases(case_ids), workbook, users=users, groups=groups, process=process) export_users(users, workbook) workbook.close() return FileWrapper(open(path))
def _es_extra_filters(self): truthy_only = functools.partial(filter, None) form_values = self.all_relevant_forms.values() if form_values: yield { 'or': [ {'and': [{'term': {'xmlns.exact': f['xmlns']}}, {'term': {'app_id': f['app_id']}}]} for f in form_values ] } users_data = ExpandedMobileWorkerFilter.pull_users_and_groups( self.domain, self.request, True, True) all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw') if not all_mobile_workers_selected or users_data.admin_and_demo_users: yield { 'terms': { 'form.meta.userID': truthy_only( u.user_id for u in users_data.combined_users ) } } else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) yield { 'not': { 'terms': { 'form.meta.userID': truthy_only( user.user_id for user in negated_ids ) } } } props = truthy_only(self.request.GET.get('form_data', '').split(',')) for prop in props: yield { 'term': {'__props_for_querying': prop.lower()} }
def _get_users_filter(self, mobile_user_and_group_slugs): truthy_only = functools.partial(filter, None) users_data = EMWF.pull_users_and_groups(self.domain, mobile_user_and_group_slugs, include_inactive=True) selected_user_types = EMWF.selected_user_types( mobile_user_and_group_slugs) all_mobile_workers_selected = HQUserType.REGISTERED in selected_user_types if not all_mobile_workers_selected or users_data.admin_and_demo_users: return form_es.user_id( truthy_only(u.user_id for u in users_data.combined_users)) else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) return es_filters.NOT( form_es.user_id( truthy_only(user.user_id for user in negated_ids)))
def _get_users_filter(self, mobile_user_and_group_slugs): truthy_only = functools.partial(filter, None) users_data = EMWF.pull_users_and_groups( self.domain, mobile_user_and_group_slugs, include_inactive=True ) selected_user_types = EMWF.selected_user_types(mobile_user_and_group_slugs) all_mobile_workers_selected = HQUserType.REGISTERED in selected_user_types if not all_mobile_workers_selected or users_data.admin_and_demo_users: return form_es.user_id(truthy_only( u.user_id for u in users_data.combined_users )) else: negated_ids = util.get_all_users_by_domain( self.domain, user_filter=HQUserType.all_but_users(), simplified=True, ) return es_filters.NOT(form_es.user_id(truthy_only( user.user_id for user in negated_ids )))