def _get_user_or_group_filter(self):
group = self._get_group()
if group:
# filter by groups
return SerializableFunction(group_filter, group=group)
# filter by users
return SerializableFunction(users_filter,
users=self._get_filtered_users())
def filter(self):
f = SerializableFunction()
if self.app_id is not None:
f.add(reports.util.app_export_filter, app_id=self.app_id)
if not self.include_errors:
f.add(couchforms.filters.instances)
return f
def filter(self):
user_ids = set(CouchUser.ids_by_domain(self.domain))
user_ids.update(CouchUser.ids_by_domain(self.domain, is_active=False))
f = SerializableFunction(form_matches_users, users=user_ids)
if self.app_id is not None:
f.add(reports.util.app_export_filter, app_id=self.app_id)
if not self.include_errors:
f.add(couchforms.filters.instances)
return f
def get_case_filter(self):
group = self._get_group()
if group:
return SerializableFunction(case_group_filter, group=group)
case_sharing_groups = [g.get_id for g in
Group.get_case_sharing_groups(self.domain_object.name)]
return SerializableFunction(case_users_filter,
users=self._get_filtered_users(),
groups=case_sharing_groups)
def download_cases(request, domain):
include_closed = json.loads(request.GET.get('include_closed', 'false'))
format = Format.from_format(request.GET.get('format') or Format.XLS_2007)
group = request.GET.get('group', None)
user_filter, _ = FilterUsersField.get_user_filter(request)
async = request.GET.get('async') == 'true'
kwargs = {
'domain': domain,
'include_closed': include_closed,
'format': format,
'group': group,
'user_filter': user_filter,
}
payload_func = SerializableFunction(generate_case_export_payload, **kwargs)
content_disposition = "attachment; filename={domain}_data.{ext}".format(
domain=domain, ext=format.extension)
mimetype = "%s" % format.mimetype
def generate_payload(payload_func):
if async:
download = DownloadBase()
a_task = prepare_download.delay(download.download_id, payload_func,
content_disposition, mimetype)
download.set_task(a_task)
return download.get_start_response()
else:
payload = payload_func()
response = HttpResponse(payload)
response['Content-Type'] = mimetype
response['Content-Disposition'] = content_disposition
return response
return generate_payload(payload_func)
def get_form_filter(self):
form_filter = SerializableFunction(app_export_filter, app_id=None)
datespan_filter = self._get_datespan_filter()
if datespan_filter:
form_filter &= datespan_filter
form_filter &= self._get_user_or_group_filter()
return form_filter
def setUp(self):
self.db = get_db('couchexport')
self.custom_export = SavedExportSchema.wrap({
'type':
'demo',
'default_format':
Format.JSON,
'index':
json.dumps(['test_custom']),
'tables': [{
'index':
'#',
'display':
'Export',
'columns': [{
'index': 'multi',
'display': 'Split',
'doc_type': 'SplitColumn',
'options': ['a', 'b', 'c', 'd']
}],
}]
})
self.custom_export.filter_function = SerializableFunction()
self.schema = [{
'#export_tag': ['string'],
'tag': 'string',
'multi': 'string'
}]
def generate_export_objects(self, export_tags):
if self.safe_only:
return []
self.export_objects = []
for schema_index in export_tags:
self.export_objects.append(
DefaultExportSchema(index=schema_index,
filter_function=SerializableFunction()))
def create_export_filter(request, domain, export_type='form'):
from corehq.apps.reports.fields import FilterUsersField
app_id = request.GET.get('app_id', None)
group, users = get_group_params(domain, **json_request(request.GET))
user_filters, use_user_filters = FilterUsersField.get_user_filter(request)
if export_type == 'case':
if user_filters and use_user_filters:
users_matching_filter = map(
lambda x: x.get('user_id'),
get_all_users_by_domain(domain,
user_filter=user_filters,
simplified=True))
filter = SerializableFunction(case_users_filter,
users=users_matching_filter)
else:
filter = SerializableFunction(case_group_filter, group=group)
else:
filter = SerializableFunction(instances) & SerializableFunction(
app_export_filter, app_id=app_id)
filter &= SerializableFunction(datespan_export_filter,
datespan=request.datespan)
if user_filters and use_user_filters:
users_matching_filter = map(
lambda x: x.get('user_id'),
get_all_users_by_domain(domain,
user_filter=user_filters,
simplified=True))
filter &= SerializableFunction(users_filter,
users=users_matching_filter)
else:
filter &= SerializableFunction(group_filter, group=group)
return filter
def export_data(req, domain):
"""
Download all data for a couchdbkit model
"""
try:
export_tag = json.loads(req.GET.get("export_tag", "null") or "null")
except ValueError:
return HttpResponseBadRequest()
group, users = util.get_group_params(domain, **json_request(req.GET))
include_errors = string_to_boolean(req.GET.get("include_errors", False))
kwargs = {
"format": req.GET.get("format", Format.XLS_2007),
"previous_export_id": req.GET.get("previous_export", None),
"filename": export_tag,
"use_cache": string_to_boolean(req.GET.get("use_cache", "True")),
"max_column_size": int(req.GET.get("max_column_size", 2000)),
"separator": req.GET.get("separator", "|")
}
user_filter, _ = UserTypeFilter.get_user_filter(req)
if user_filter:
filtered_users = users_matching_filter(domain, user_filter)
def _ufilter(user):
try:
return user['form']['meta']['userID'] in filtered_users
except KeyError:
return False
filter = _ufilter
else:
filter = SerializableFunction(util.group_filter, group=group)
errors_filter = instances if not include_errors else None
kwargs['filter'] = couchexport.util.intersect_functions(
filter, errors_filter)
if kwargs['format'] == 'raw':
resp = export_raw_data([domain, export_tag], filename=export_tag)
else:
try:
resp = export_data_shared([domain, export_tag], **kwargs)
except CouchExportException as e:
return HttpResponseBadRequest(e)
if resp:
return resp
else:
messages.error(
req,
"Sorry, there was no data found for the tag '%s'." % export_tag)
next = req.GET.get("next", "")
if not next:
next = export.ExcelExportReport.get_url(domain=domain)
return HttpResponseRedirect(next)
def export_cols():
for col in cols:
transform = self.request.POST.get('%s transform' % col) or None
if transform:
transform = SerializableFunction.loads(transform)
yield ExportColumn(
index=col,
display=self.request.POST["%s display" % col],
transform=transform
)
def create_export_filter(request, domain, export_type='form'):
from corehq.apps.reports.filters.users import UserTypeFilter
app_id = request.GET.get('app_id', None)
group, users = get_group_params(domain, **json_request(request.GET))
user_filters, use_user_filters = UserTypeFilter.get_user_filter(request)
if export_type == 'case':
if user_filters and use_user_filters:
filtered_users = users_matching_filter(domain, user_filters)
filter = SerializableFunction(case_users_filter,
users=filtered_users)
else:
filter = SerializableFunction(case_group_filter, group=group)
else:
filter = SerializableFunction(app_export_filter, app_id=app_id)
datespan = request.datespan
if datespan.is_valid():
datespan.set_timezone(get_timezone(request.couch_user, domain))
filter &= SerializableFunction(datespan_export_filter, datespan=datespan)
if user_filters and use_user_filters:
filtered_users = users_matching_filter(domain, user_filters)
filter &= SerializableFunction(users_filter, users=filtered_users)
else:
filter &= SerializableFunction(group_filter, group=group)
return filter
def create_export_filter(request, domain, export_type='form'):
request_obj = request.POST if request.method == 'POST' else request.GET
from corehq.apps.reports.filters.users import UserTypeFilter
app_id = request_obj.get('app_id', None)
user_filters, use_user_filters = UserTypeFilter.get_user_filter(request)
use_user_filters &= bool(user_filters)
group = None if use_user_filters else get_group(
**json_request(request_obj))
if export_type == 'case':
if use_user_filters:
groups = [g.get_id for g in Group.get_case_sharing_groups(domain)]
filtered_users = users_matching_filter(domain, user_filters)
filter = SerializableFunction(case_users_filter,
users=filtered_users,
groups=groups)
else:
filter = SerializableFunction(case_group_filter, group=group)
else:
filter = SerializableFunction(app_export_filter, app_id=app_id)
datespan = request.datespan
if datespan.is_valid():
datespan.set_timezone(
get_timezone_for_user(request.couch_user, domain))
filter &= SerializableFunction(datespan_export_filter,
datespan=datespan)
if use_user_filters:
filtered_users = users_matching_filter(domain, user_filters)
filter &= SerializableFunction(users_filter, users=filtered_users)
else:
filter &= SerializableFunction(group_filter, group=group)
return filter
def filter(self):
f = SerializableFunction()
if self.app_id is not None:
f.add(reports.util.app_export_filter, app_id=self.app_id)
if not self.include_errors:
f.add(couchforms.filters.instances)
return f
def filter(self):
user_ids = set(CouchUser.ids_by_domain(self.domain))
user_ids.update(CouchUser.ids_by_domain(self.domain, is_active=False))
def _top_level_filter(form):
# careful, closures used
return form_matches_users(form, user_ids) or is_commconnect_form(form)
f = SerializableFunction(_top_level_filter)
if self.app_id is not None:
f.add(reports.util.app_export_filter, app_id=self.app_id)
if not self.include_errors:
f.add(couchforms.filters.instances)
actual = SerializableFunction(default_form_filter, filter=f)
return actual
pass
def deid_date(val, doc, key_path='form/case/@case_id|form/case/case_id|_id'):
key = JSONPath(key_path).search(doc)
if not key:
return None
offset = DeidGenerator(key, 'date').random_number(-31, 32)
orig_date = string_to_datetime(val)
return (orig_date + timedelta(days=offset)).date()
def deid_remove(val, doc):
return Ellipsis
def deid_map(doc, config):
doc_copy = doc.copy()
for key in config:
parts = key.split('/')
final_part = parts.pop()
ctx = doc_copy
for part in parts:
ctx = ctx[part]
if config[key]:
ctx[final_part] = config[key](ctx[final_part], doc)
if ctx[final_part] == Ellipsis:
del ctx[final_part]
return doc_copy
OPTIONS = map(lambda (name, f): (name, SerializableFunction.to_path(f)), [
('Sensitive ID', deid_ID),
('Sensitive Date', deid_date),
])
def _get_datespan_filter(self):
datespan = self._get_datespan()
if datespan.is_valid():
datespan.set_timezone(self.timezone)
return SerializableFunction(datespan_export_filter,
datespan=datespan)
else:
if not async:
# this function doesn't support synchronous export without a custom export object
# if we ever want that (i.e. for HTML Preview) then we just need to give
# FakeSavedExportSchema a download_data function (called below)
return HttpResponseBadRequest()
try:
export_tag = json.loads(req.get("export_tag", "null") or "null")
except ValueError:
return HttpResponseBadRequest()
assert (export_tag[0] == domain)
# hack - also filter instances here rather than mess too much with trying to make this
# look more like a FormExportSchema
export_class = FakeSavedExportSchema
if export_type == 'form':
filter &= SerializableFunction(instances)
export_class = FakeFormExportSchema
export_object = export_class(index=export_tag)
if export_type == 'form':
_filter = filter
filter = SerializableFunction(default_form_filter, filter=_filter)
if not filename:
filename = export_object.name
filename += ' ' + date.today().isoformat()
if async:
return export_object.export_data_async(
filter=filter,
def filter(self):
return SerializableFunction(default_case_filter)
key = JSONPath(key_path).search(doc)
if not key:
return None
offset = DeidGenerator(key, 'date').random_number(-31, 32)
orig_date = string_to_datetime(val)
return (orig_date + timedelta(days=offset)).date()
def deid_remove(val, doc):
return Ellipsis
def deid_map(doc, config):
doc_copy = doc.copy()
for key in config:
parts = key.split('/')
final_part = parts.pop()
ctx = doc_copy
for part in parts:
ctx = ctx[part]
if config[key]:
ctx[final_part] = config[key](ctx[final_part], doc)
if ctx[final_part] == Ellipsis:
del ctx[final_part]
return doc_copy
OPTIONS = map(lambda (name, f): (name, SerializableFunction.to_path(f)), [
('Sensitive ID', deid_ID),
('Sensitive Date', deid_date),
])
def transform(self):
return SerializableFunction()
