def generate_case_export_payload(domain, include_closed, format, group, user_filter, process=None):
"""
Returns a FileWrapper object, which only the file backend in django-soil supports
"""
status = "all" if include_closed else "open"
case_ids = CommCareCase.get_all_cases(domain, status=status, wrapper=lambda r: r["id"])
class stream_cases(object):
def __init__(self, all_case_ids):
self.all_case_ids = all_case_ids
def __iter__(self):
for case_ids in chunked(self.all_case_ids, 500):
for case in wrapped_docs(CommCareCase, case_ids):
yield case
def __len__(self):
return len(self.all_case_ids)
# todo deal with cached user dict here
users = get_all_users_by_domain(domain, group=group, user_filter=user_filter)
groups = Group.get_case_sharing_groups(domain)
fd, path = tempfile.mkstemp()
with os.fdopen(fd, "wb") as file:
workbook = WorkBook(file, format)
export_cases_and_referrals(
domain, stream_cases(case_ids), workbook, users=users, groups=groups, process=process
)
export_users(users, workbook)
workbook.close()
return FileWrapper(open(path))
def export_all_form_metadata_async(req, domain):
datespan = req.datespan if req.GET.get("startdate") and req.GET.get("enddate") else None
group_id = req.GET.get("group")
ufilter = UserTypeFilter.get_user_filter(req)[0]
users = util.get_all_users_by_domain(
domain=domain,
group=group_id,
user_filter=ufilter,
simplified=True,
include_inactive=True
)
user_ids = filter(None, [u["user_id"] for u in users])
format = req.GET.get("format", Format.XLS_2007)
filename = "%s_forms" % domain
download = DownloadBase()
download.set_task(create_metadata_export.delay(
download.download_id,
domain,
format=format,
filename=filename,
datespan=datespan,
user_ids=user_ids,
))
return download.get_start_response()
def export_all_form_metadata_async(req, domain):
datespan = req.datespan if req.GET.get("startdate") and req.GET.get(
"enddate") else None
group_id = req.GET.get("group")
ufilter = UserTypeFilter.get_user_filter(req)[0]
users = list(
util.get_all_users_by_domain(domain=domain,
group=group_id,
user_filter=ufilter,
simplified=True))
user_ids = filter(None, [u["user_id"] for u in users])
format = req.GET.get("format", Format.XLS_2007)
filename = "%s_forms" % domain
download = DownloadBase()
download.set_task(
create_metadata_export.delay(
download.download_id,
domain,
format=format,
filename=filename,
datespan=datespan,
user_ids=user_ids,
))
return download.get_start_response()
def es_query(self):
if not getattr(self, 'es_response', None):
q = {
"query": {
"range": {
self.time_field: {
"from": self.datespan.startdate_param,
"to": self.datespan.enddate_param,
"include_upper": False}}},
"filter": {"and": ADD_TO_ES_FILTER["forms"][:]}}
xmlnss = filter(None, [f["xmlns"] for f in self.all_relevant_forms.values()])
if xmlnss:
q["filter"]["and"].append({"terms": {"xmlns.exact": xmlnss}})
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(self.domain, self.request, True, True)
if "t__0" not in self.request.GET.getlist("emw") or users_data["admin_and_demo_users"]:
q["filter"]["and"].append(
{"terms": {"form.meta.userID": filter(None, [u["user_id"] for u in users_data["combined_users"]])}})
else:
negated_ids = util.get_all_users_by_domain(self.domain, user_filter=HQUserType.all_but_users(), simplified=True)
ids = filter(None, [user['user_id'] for user in negated_ids])
q["filter"]["and"].append({"not": {"terms": {"form.meta.userID": ids}}})
for cp in filter(None, self.request.GET.get('form_data', "").split(",")):
q["filter"]["and"].append({"term": {"__props_for_querying": cp.lower()}})
q["sort"] = self.get_sorting_block() if self.get_sorting_block() else [{self.time_field : {"order": "desc"}}]
self.es_response = es_query(params={"domain.exact": self.domain}, q=q, es_url=XFORM_INDEX + '/xform/_search',
start_at=self.pagination.start, size=self.pagination.count)
return self.es_response
def users(self):
"""
todo: cache this baby
"""
if self._users is None:
self._users = util.get_all_users_by_domain(self.domain,
group=self.group_name, individual=self.individual, user_filter=self.user_filter, simplified=True)
return self._users
def user_cacher(domain, cache_key,
current_cache=None, refresh_stale=1800, cache_timeout=3600, **kwargs):
from corehq.apps.reports.util import get_all_users_by_domain
data = get_all_users_by_domain(domain, **kwargs)
_cache_data(data, cache_key,
current_cache=current_cache,
refresh_stale=refresh_stale,
cache_timeout=cache_timeout)
def get_all_users_by_domain(self, group=None, user_ids=None, user_filter=None, simplified=False):
return list(util.get_all_users_by_domain(
domain=self.domain,
group=group,
user_ids=user_ids,
user_filter=user_filter,
simplified=False, # override simplified to False
CommCareUser=self.CommCareUser
))
def get_all_users_by_domain(self, group=None, user_ids=None, user_filter=None, simplified=False):
return list(util.get_all_users_by_domain(
domain=self.domain,
group=group,
user_ids=user_ids,
user_filter=user_filter,
simplified=simplified,
CommCareUser=self.CommCareUser
))
def get_all_users_by_domain(self, group=None, individual=None, user_filter=None, simplified=False):
return list(util.get_all_users_by_domain(
domain=self.domain,
group=group,
individual=individual,
user_filter=user_filter,
simplified=simplified,
CommCareUser=self.CommCareUser
))
def report_context(self):
context = super(CaseReassignmentInterface, self).report_context
active_users = util.get_all_users_by_domain(self.domain, user_filter=HQUserType.use_defaults(), simplified=True)
context.update(
users=[dict(ownerid=user.get('user_id'), name=user.get('username_in_report'), type="user")
for user in active_users],
groups=[dict(ownerid=group.get_id, name=group.name, type="group")
for group in self.all_case_sharing_groups]
)
return context
def users(self):
group = self.group if not self.user_id else None
user_ids = (self.user_id,)
users = list(
util.get_all_users_by_domain(
domain=self.domain, user_ids=user_ids, group=group, simplified=True, CommCareUser=CommCareUserMemoizer()
)
)
return sorted(users, key=itemgetter("raw_username"))
def _es_extra_filters(self):
if FormsByApplicationFilter.has_selections(self.request):
def form_filter(form):
app_id = form.get('app_id', None)
if app_id and app_id != MISSING_APP_ID:
return {'and': [{'term': {'xmlns.exact': form['xmlns']}},
{'term': {'app_id': app_id}}]}
return {'term': {'xmlns.exact': form['xmlns']}}
form_values = self.all_relevant_forms.values()
if form_values:
yield {'or': [form_filter(f) for f in form_values]}
truthy_only = functools.partial(filter, None)
mobile_user_and_group_slugs = self.request.GET.getlist(ExpandedMobileWorkerFilter.slug)
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain,
mobile_user_and_group_slugs,
include_inactive=True
)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID': truthy_only(
u.user_id for u in users_data.combined_users
)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID': truthy_only(
user.user_id for user in negated_ids
)
}
}
}
props = truthy_only(self.request.GET.get('form_data', '').split(','))
for prop in props:
yield {
'term': {'__props_for_querying': prop}
}
if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types(mobile_user_and_group_slugs):
yield {
'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}}
}
def users(self):
group = self.group if not self.user_id else None
user_ids = (self.user_id, )
users = list(
util.get_all_users_by_domain(domain=self.domain,
user_ids=user_ids,
group=group,
simplified=True,
CommCareUser=CommCareUserMemoizer()))
return sorted(users, key=itemgetter('raw_username'))
def _es_extra_filters(self):
if FormsByApplicationFilter.has_selections(self.request):
def form_filter(form):
app_id = form.get('app_id', None)
if app_id and app_id != MISSING_APP_ID:
return {
'and': [{
'term': {
'xmlns.exact': form['xmlns']
}
}, {
'term': {
'app_id': app_id
}
}]
}
return {'term': {'xmlns.exact': form['xmlns']}}
form_values = self.all_relevant_forms.values()
if form_values:
yield {'or': [form_filter(f) for f in form_values]}
truthy_only = functools.partial(filter, None)
mobile_user_and_group_slugs = self.request.GET.getlist(
ExpandedMobileWorkerFilter.slug)
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain, mobile_user_and_group_slugs, include_inactive=True)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID':
truthy_only(u.user_id for u in users_data.combined_users)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID':
truthy_only(user.user_id for user in negated_ids)
}
}
}
if HQUserType.UNKNOWN not in ExpandedMobileWorkerFilter.selected_user_types(
mobile_user_and_group_slugs):
yield {'not': {'term': {'xmlns.exact': SYSTEM_FORM_XMLNS}}}
def export_data(req, domain):
"""
Download all data for a couchdbkit model
"""
try:
export_tag = json.loads(req.GET.get("export_tag", "null") or "null")
except ValueError:
return HttpResponseBadRequest()
group, users = util.get_group_params(domain, **json_request(req.GET))
include_errors = string_to_boolean(req.GET.get("include_errors", False))
kwargs = {"format": req.GET.get("format", Format.XLS_2007),
"previous_export_id": req.GET.get("previous_export", None),
"filename": export_tag,
"use_cache": string_to_boolean(req.GET.get("use_cache", "True")),
"max_column_size": int(req.GET.get("max_column_size", 2000)),
"separator": req.GET.get("separator", "|")}
user_filter, _ = FilterUsersField.get_user_filter(req)
if user_filter:
users_matching_filter = map(lambda x: x.get('user_id'),
get_all_users_by_domain(domain, user_filter=user_filter, simplified=True))
def _ufilter(user):
try:
return user['form']['meta']['userID'] in users_matching_filter
except KeyError:
return False
filter = _ufilter
else:
filter = SerializableFunction(util.group_filter, group=group)
errors_filter = instances if not include_errors else None
kwargs['filter'] = couchexport.util.intersect_functions(filter, errors_filter)
if kwargs['format'] == 'raw':
resp = export_raw_data([domain, export_tag], filename=export_tag)
else:
try:
resp = export_data_shared([domain,export_tag], **kwargs)
except UnsupportedExportFormat as e:
return HttpResponseBadRequest(e)
if resp:
return resp
else:
messages.error(req, "Sorry, there was no data found for the tag '%s'." % export_tag)
next = req.GET.get("next", "")
if not next:
next = export.ExcelExportReport.get_url(domain=domain)
return HttpResponseRedirect(next)
def _es_extra_filters(self):
if FormsByApplicationFilter.has_selections(self.request):
def form_filter(form):
app_id = form.get('app_id', None)
if app_id and app_id != MISSING_APP_ID:
return {
'and': [{
'term': {
'xmlns.exact': form['xmlns']
}
}, {
'term': {
'app_id': app_id
}
}]
}
return {'term': {'xmlns.exact': form['xmlns']}}
form_values = self.all_relevant_forms.values()
if form_values:
yield {'or': [form_filter(f) for f in form_values]}
truthy_only = functools.partial(filter, None)
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain, self.request, True, True, include_inactive=True)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID':
truthy_only(u.user_id for u in users_data.combined_users)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID':
truthy_only(user.user_id for user in negated_ids)
}
}
}
props = truthy_only(self.request.GET.get('form_data', '').split(','))
for prop in props:
yield {'term': {'__props_for_querying': prop.lower()}}
def export_data(req, domain):
"""
Download all data for a couchdbkit model
"""
try:
export_tag = json.loads(req.GET.get("export_tag", "null") or "null")
except ValueError:
return HttpResponseBadRequest()
group, users = util.get_group_params(domain, **json_request(req.GET))
include_errors = string_to_boolean(req.GET.get("include_errors", False))
kwargs = {"format": req.GET.get("format", Format.XLS_2007),
"previous_export_id": req.GET.get("previous_export", None),
"filename": export_tag,
"use_cache": string_to_boolean(req.GET.get("use_cache", "True")),
"max_column_size": int(req.GET.get("max_column_size", 2000)),
"separator": req.GET.get("separator", "|")}
user_filter, _ = UserTypeFilter.get_user_filter(req)
if user_filter:
users_matching_filter = map(lambda x: x.get('user_id'),
get_all_users_by_domain(domain, user_filter=user_filter, simplified=True))
def _ufilter(user):
try:
return user['form']['meta']['userID'] in users_matching_filter
except KeyError:
return False
filter = _ufilter
else:
filter = SerializableFunction(util.group_filter, group=group)
errors_filter = instances if not include_errors else None
kwargs['filter'] = couchexport.util.intersect_functions(filter, errors_filter)
if kwargs['format'] == 'raw':
resp = export_raw_data([domain, export_tag], filename=export_tag)
else:
try:
resp = export_data_shared([domain, export_tag], **kwargs)
except CouchExportException as e:
return HttpResponseBadRequest(e)
if resp:
return resp
else:
messages.error(req, "Sorry, there was no data found for the tag '%s'." % export_tag)
next = req.GET.get("next", "")
if not next:
next = export.ExcelExportReport.get_url(domain=domain)
return HttpResponseRedirect(next)
def pull_users_and_groups(cls, domain, request, simplified_users=False, combined=False, CommCareUser=CommCareUser):
emws = request.GET.getlist(cls.slug)
users = []
user_ids = [u[3:] for u in filter(lambda s: s.startswith("u__"), emws)]
if user_ids or "t__0" in emws:
users = util.get_all_users_by_domain(domain=domain, user_ids=user_ids, simplified=simplified_users,
CommCareUser=CommCareUser)
user_type_ids = [int(t[3:]) for t in filter(lambda s: s.startswith("t__"), emws)]
user_filter = tuple([HQUserToggle(id, id in user_type_ids) for id in range(4)])
other_users = util.get_all_users_by_domain(domain=domain, user_filter=user_filter, simplified=simplified_users,
CommCareUser=CommCareUser)
group_ids = [g[3:] for g in filter(lambda s: s.startswith("g__"), emws)]
groups = [Group.get(g) for g in group_ids]
ret = {
"users": users + other_users,
"admin_and_demo_users": other_users,
"groups": groups,
}
if combined:
user_dict = {}
for group in groups:
user_dict["%s|%s" % (group.name, group._id)] = util.get_all_users_by_domain(
group=group,
simplified=simplified_users
)
users_in_groups = [user for sublist in user_dict.values() for user in sublist]
ret["users_by_group"] = user_dict
ret["combined_users"] = remove_dups(ret["users"] + users_in_groups, "user_id")
return ret
def generate_case_export_payload(domain,
include_closed,
format,
group,
user_filter,
process=None):
"""
Returns a FileWrapper object, which only the file backend in django-soil supports
"""
status = 'all' if include_closed else 'open'
case_ids = CommCareCase.get_all_cases(domain,
status=status,
wrapper=lambda r: r['id'])
class stream_cases(object):
def __init__(self, all_case_ids):
self.all_case_ids = all_case_ids
def __iter__(self):
for case_ids in chunked(self.all_case_ids, 500):
for case in wrapped_docs(CommCareCase, case_ids):
yield case
def __len__(self):
return len(self.all_case_ids)
# todo deal with cached user dict here
users = get_all_users_by_domain(domain,
group=group,
user_filter=user_filter)
groups = Group.get_case_sharing_groups(domain)
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as file:
workbook = WorkBook(file, format)
export_cases_and_referrals(domain,
stream_cases(case_ids),
workbook,
users=users,
groups=groups,
process=process)
export_users(users, workbook)
workbook.close()
return FileWrapper(open(path))
def _es_extra_filters(self):
truthy_only = functools.partial(filter, None)
form_values = self.all_relevant_forms.values()
if form_values:
yield {
'or': [
{'and': [{'term': {'xmlns.exact': f['xmlns']}},
{'term': {'app_id': f['app_id']}}]}
for f in form_values
]
}
users_data = ExpandedMobileWorkerFilter.pull_users_and_groups(
self.domain, self.request, True, True)
all_mobile_workers_selected = 't__0' in self.request.GET.getlist('emw')
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
yield {
'terms': {
'form.meta.userID': truthy_only(
u.user_id for u in users_data.combined_users
)
}
}
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
yield {
'not': {
'terms': {
'form.meta.userID': truthy_only(
user.user_id for user in negated_ids
)
}
}
}
props = truthy_only(self.request.GET.get('form_data', '').split(','))
for prop in props:
yield {
'term': {'__props_for_querying': prop.lower()}
}
def _get_users_filter(self, mobile_user_and_group_slugs):
truthy_only = functools.partial(filter, None)
users_data = EMWF.pull_users_and_groups(self.domain,
mobile_user_and_group_slugs,
include_inactive=True)
selected_user_types = EMWF.selected_user_types(
mobile_user_and_group_slugs)
all_mobile_workers_selected = HQUserType.REGISTERED in selected_user_types
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
return form_es.user_id(
truthy_only(u.user_id for u in users_data.combined_users))
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
return es_filters.NOT(
form_es.user_id(
truthy_only(user.user_id for user in negated_ids)))
def _get_users_filter(self, mobile_user_and_group_slugs):
truthy_only = functools.partial(filter, None)
users_data = EMWF.pull_users_and_groups(
self.domain,
mobile_user_and_group_slugs,
include_inactive=True
)
selected_user_types = EMWF.selected_user_types(mobile_user_and_group_slugs)
all_mobile_workers_selected = HQUserType.REGISTERED in selected_user_types
if not all_mobile_workers_selected or users_data.admin_and_demo_users:
return form_es.user_id(truthy_only(
u.user_id for u in users_data.combined_users
))
else:
negated_ids = util.get_all_users_by_domain(
self.domain,
user_filter=HQUserType.all_but_users(),
simplified=True,
)
return es_filters.NOT(form_es.user_id(truthy_only(
user.user_id for user in negated_ids
)))
