def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
if post_data['presave'] and not self.allow_daily_saved:
raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports"))
if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard:
raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard"))
if custom_export_json["is_safe"] and not self.allow_deid:
raise BadExportConfiguration(_("This user does not have permission to create a de-identified export"))
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
if post_data['presave'] and not self.allow_daily_saved:
raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports"))
if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard:
raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard"))
if custom_export_json["is_safe"] and not self.allow_deid:
raise BadExportConfiguration(_("This user does not have permission to create a de-identified export"))
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
schema = ExportSchema.get(self.request.POST["schema"])
self.custom_export.index = schema.index
self.custom_export.schema_id = self.request.POST["schema"]
self.custom_export.name = self.request.POST["name"]
self.custom_export.default_format = self.request.POST["format"] or Format.XLS_2007
self.custom_export.is_safe = bool(self.request.POST.get('is_safe'))
self.presave = bool(self.request.POST.get('presave'))
table = self.request.POST["table"]
cols = self.request.POST['order'].strip().split()
@list
@inline
def export_cols():
for col in cols:
transform = self.request.POST.get('%s transform' % col) or None
if transform:
transform = SerializableFunction.loads(transform)
yield ExportColumn(
index=col,
display=self.request.POST["%s display" % col],
transform=transform
)
export_table = ExportTable(index=table, display=self.request.POST["name"], columns=export_cols)
self.custom_export.tables = [export_table]
self.custom_export.order = cols
table_dict = dict([t.index, t] for t in self.custom_export.tables)
if table in table_dict:
table_dict[table].columns = export_cols
else:
self.custom_export.tables.append(ExportTable(index=table,
display=self.custom_export.name,
columns=export_cols))
if self.export_type == 'form':
self.custom_export.include_errors = bool(self.request.POST.get("include-errors"))
self.custom_export.app_id = self.request.POST.get('app_id')
self.custom_export.save()
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def __init__(self, request, domain, export_id=None, minimal=False):
self.request = request
self.domain = domain
self.presave = False
self.transform_dates = False
self.creating_new_export = not bool(export_id)
self.minimal = minimal
if export_id:
self.custom_export = self.ExportSchemaClass.get(export_id)
# also update the schema to include potential new stuff
self.custom_export.update_schema()
# enable configuring saved exports from this page
saved_group = HQGroupExportConfiguration.get_for_domain(
self.domain)
self.presave = export_id in saved_group.custom_export_ids
self.export_stock = self.has_stock_column()
try:
assert self.custom_export.doc_type == 'SavedExportSchema', 'bad export doc type'
assert self.custom_export.type == self.export_type, 'wrong export type specified'
assert self.custom_export.index[
0] == domain, 'bad export doc domain'
except AssertionError as e:
raise BadExportConfiguration(str(e))
else:
self.custom_export = self.ExportSchemaClass(type=self.export_type)
self.export_stock = False
def __init__(self, request, domain, export_id=None):
self.request = request
self.domain = domain
self.export_type = request.GET.get('type', 'form')
self.presave = False
if self.export_type == 'form':
self.ExportSchemaClass = FormExportSchema
else:
self.ExportSchemaClass = SavedExportSchema
if export_id:
self.custom_export = self.ExportSchemaClass.get(export_id)
# also update the schema to include potential new stuff
self.custom_export.update_schema()
# enable configuring saved exports from this page
saved_group = HQGroupExportConfiguration.get_for_domain(self.domain)
self.presave = export_id in saved_group.custom_export_ids
assert(self.custom_export.doc_type == 'SavedExportSchema')
assert(self.custom_export.type == self.export_type)
assert(self.custom_export.index[0] == domain)
else:
self.custom_export = self.ExportSchemaClass(type=self.export_type)
if self.export_type == 'form':
self.custom_export.app_id = request.GET.get('app_id')
def __init__(self, request, domain, export_id=None, minimal=False):
self.request = request
self.domain = domain
self.presave = False
self.transform_dates = False
self.creating_new_export = not bool(export_id)
self.minimal = minimal
if export_id:
self.custom_export = self.ExportSchemaClass.get(export_id)
# also update the schema to include potential new stuff
self.custom_export.update_schema()
# enable configuring saved exports from this page
saved_group = HQGroupExportConfiguration.get_for_domain(self.domain)
self.presave = export_id in saved_group.custom_export_ids
self.export_stock = self.has_stock_column()
try:
assert self.custom_export.doc_type == 'SavedExportSchema', 'bad export doc type'
assert self.custom_export.type == self.export_type, 'wrong export type specified'
assert self.custom_export.index[0] == domain, 'bad export doc domain'
except AssertionError, e:
raise BadExportConfiguration(str(e))
def hq_update_saved_export(req, domain):
group_id = req.POST["group_export_id"]
index = int(req.POST["index"])
group_config = HQGroupExportConfiguration.get(group_id)
assert domain == group_config.domain
config, schema = group_config.all_exports[index]
rebuild_export(config, schema, "couch")
messages.success(req, _("The data for {} has been refreshed!").format(config.name))
return HttpResponseRedirect(reverse(DataInterfaceDispatcher.name(), args=[domain, req.POST["report_slug"]]))
def get_all_hq_group_export_configs():
from corehq.apps.reports.models import HQGroupExportConfiguration
return imap(
HQGroupExportConfiguration.wrap,
get_all_docs_with_doc_types(
HQGroupExportConfiguration.get_db(),
('HQGroupExportConfiguration',)
)
)
def report_context(self):
context = super(CaseExportReport, self).report_context
case_types = get_case_types_for_domain(self.domain)
groups = HQGroupExportConfiguration.by_domain(self.domain)
context.update(
case_types=case_types,
group_exports=[group.case_exports for group in groups if group.case_exports],
report_slug=self.slug,
)
context["case_format"] = self.request.GET.get("case_format") or "csv"
return context
def report_context(self):
context = super(CaseExportReport, self).report_context
case_types = get_case_types_for_domain(self.domain)
groups = HQGroupExportConfiguration.by_domain(self.domain)
context.update(
case_types=case_types,
group_exports=[group.case_exports for group in groups
if group.case_exports],
report_slug=self.slug,
)
context['case_format'] = self.request.GET.get('case_format') or 'csv'
return context
def hq_update_saved_export(req, domain):
group_id = req.POST['group_export_id']
index = int(req.POST['index'])
group_config = HQGroupExportConfiguration.get(group_id)
assert domain == group_config.domain
config, schema = group_config.all_exports[index]
rebuild_export(config, schema, 'couch')
messages.success(
req,
_('The data for {} has been refreshed!').format(config.name))
return HttpResponseRedirect(
reverse(DataInterfaceDispatcher.name(),
args=[domain, req.POST['report_slug']]))
def report_context(self):
context = super(CaseExportReport, self).report_context
cases = get_db().view("hqcase/types_by_domain",
startkey=[self.domain],
endkey=[self.domain, {}],
reduce=True,
group=True,
group_level=2).all()
groups = HQGroupExportConfiguration.by_domain(self.domain)
context.update(
case_types=[case['key'][1] for case in cases],
group_exports=[group.case_exports for group in groups
if group.case_exports],
)
return context
def report_context(self):
context = super(CaseExportReport, self).report_context
cases = get_db().view("hqcase/types_by_domain",
startkey=[self.domain],
endkey=[self.domain, {}],
reduce=True,
group=True,
group_level=2).all()
groups = HQGroupExportConfiguration.by_domain(self.domain)
context.update(
case_types=[case['key'][1] for case in cases],
group_exports=[
group.case_exports for group in groups if group.case_exports
],
)
context['case_format'] = self.request.GET.get('case_format') or 'csv'
return context
def report_context(self):
context = super(CaseExportReport, self).report_context
cases = CommCareCase.get_db().view("hqcase/types_by_domain",
startkey=[self.domain],
endkey=[self.domain, {}],
reduce=True,
group=True,
group_level=2).all()
groups = HQGroupExportConfiguration.by_domain(self.domain)
context.update(
case_types=[case['key'][1] for case in cases],
group_exports=[group.case_exports for group in groups
if group.case_exports],
report_slug=self.slug,
)
context['case_format'] = self.request.GET.get('case_format') or 'csv'
return context
def __init__(self, request, domain, export_id=None):
self.request = request
self.domain = domain
self.export_type = request.GET.get('type', 'form')
self.presave = False
if export_id:
self.custom_export = self.ExportSchemaClass.get(export_id)
# also update the schema to include potential new stuff
self.custom_export.update_schema()
# enable configuring saved exports from this page
saved_group = HQGroupExportConfiguration.get_for_domain(self.domain)
self.presave = export_id in saved_group.custom_export_ids
assert(self.custom_export.doc_type == 'SavedExportSchema')
assert(self.custom_export.type == self.export_type)
assert(self.custom_export.index[0] == domain)
else:
self.custom_export = self.ExportSchemaClass(type=self.export_type)
def __init__(self, request, domain, export_id=None):
self.request = request
self.domain = domain
self.presave = False
self.transform_dates = False
self.creating_new_export = not bool(export_id)
if export_id:
self.custom_export = self.ExportSchemaClass.get(export_id)
# also update the schema to include potential new stuff
self.custom_export.update_schema()
# enable configuring saved exports from this page
saved_group = HQGroupExportConfiguration.get_for_domain(self.domain)
self.presave = export_id in saved_group.custom_export_ids
assert(self.custom_export.doc_type == 'SavedExportSchema')
assert(self.custom_export.type == self.export_type)
assert(self.custom_export.index[0] == domain)
else:
self.custom_export = self.ExportSchemaClass(type=self.export_type)
def test_daily_saved_conversion(self, _):
# ID is from corehq/apps/export/tests/data/saved_export_schemas/case.json
self.group_config = HQGroupExportConfiguration.add_custom_export(
self.domain, '92e5f9a6624a637c2080957475cd446d')
self.group_config.save()
self.addCleanup(self.group_config.delete)
instance, _ = self._convert_case_export('case')
self.assertEqual(instance.transform_dates, True)
self.assertEqual(instance.name, 'Case Example')
self.assertEqual(instance.export_format, 'csv')
self.assertEqual(instance.is_deidentified, False)
self.assertEqual(instance.is_daily_saved_export, True)
table = instance.get_table(MAIN_TABLE)
self.assertEqual(table.label, 'Cases')
self.assertTrue(table.selected)
index, column = table.get_column([PathNode(name='DOB')], 'ExportItem',
None)
self.assertEqual(column.label, 'DOB Saved')
self.assertEqual(column.selected, True)
def test_daily_saved_conversion(self, _):
# ID is from corehq/apps/export/tests/data/saved_export_schemas/case.json
self.group_config = HQGroupExportConfiguration.add_custom_export(
self.domain, "92e5f9a6624a637c2080957475cd446d"
)
self.group_config.save()
self.addCleanup(self.group_config.delete)
instance, _ = self._convert_case_export("case")
self.assertEqual(instance.transform_dates, True)
self.assertEqual(instance.name, "Case Example")
self.assertEqual(instance.export_format, "csv")
self.assertEqual(instance.is_deidentified, False)
self.assertEqual(instance.is_daily_saved_export, True)
table = instance.get_table(MAIN_TABLE)
self.assertEqual(table.label, "Cases")
self.assertTrue(table.selected)
index, column = table.get_column([PathNode(name="DOB")], "ExportItem", None)
self.assertEqual(column.label, "DOB Saved")
self.assertEqual(column.selected, True)
def saved_exports():
for row in HQGroupExportConfiguration.view("groupexport/by_domain",
reduce=False).all():
export_for_group(row["id"], "couch")
def report_context(self):
# This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids.
# However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}]
forms = []
unknown_forms = []
for f in get_db().view("reports/forms_by_xmlns", startkey=[self.domain], endkey=[self.domain, {}], group=True):
form = f["value"]
if form.get("app_deleted") and not form.get("submissions"):
continue
if "app" in form:
form["has_app"] = True
else:
app_id = f["key"][1] or ""
form["app"] = {"id": app_id}
form["has_app"] = False
form["show_xmlns"] = True
unknown_forms.append(form)
form["current_app"] = form.get("app")
forms.append(form)
if unknown_forms:
apps = get_db().view(
"reports/forms_by_xmlns",
startkey=["^Application", self.domain],
endkey=["^Application", self.domain, {}],
reduce=False,
)
possibilities = defaultdict(list)
for app in apps:
# index by xmlns
x = app["value"]
x["has_app"] = True
possibilities[app["key"][2]].append(x)
class AppCache(dict):
def __init__(self, domain):
super(AppCache, self).__init__()
self.domain = domain
def __getitem__(self, item):
if not self.has_key(item):
try:
self[item] = get_app(app_id=item, domain=self.domain)
except Http404:
pass
return super(AppCache, self).__getitem__(item)
app_cache = AppCache(self.domain)
for form in unknown_forms:
app = None
if form["app"]["id"]:
try:
app = app_cache[form["app"]["id"]]
form["has_app"] = True
except KeyError:
form["app_does_not_exist"] = True
form["possibilities"] = possibilities[form["xmlns"]]
if form["possibilities"]:
form["duplicate"] = True
else:
if app.domain != self.domain:
logging.error("submission tagged with app from wrong domain: %s" % app.get_id)
else:
if app.copy_of:
try:
app = app_cache[app.copy_of]
form["app_copy"] = {"id": app.get_id, "name": app.name}
except KeyError:
form["app_copy"] = {"id": app.copy_of, "name": "?"}
if app.is_deleted():
form["app_deleted"] = {"id": app.get_id}
try:
app_forms = app.get_xmlns_map()[form["xmlns"]]
except AttributeError:
# it's a remote app
app_forms = None
if app_forms:
app_form = app_forms[0]
if app_form.doc_type == "UserRegistrationForm":
form["is_user_registration"] = True
else:
app_module = app_form.get_module()
form["module"] = app_module
form["form"] = app_form
form["show_xmlns"] = False
if not form.get("app_copy") and not form.get("app_deleted"):
form["no_suggestions"] = True
if app:
form["app"] = {"id": app.get_id, "name": app.name, "langs": app.langs}
else:
form["possibilities"] = possibilities[form["xmlns"]]
if form["possibilities"]:
form["duplicate"] = True
else:
form["no_suggestions"] = True
forms = sorted(
forms,
key=lambda form: (
0 if not form.get("app_deleted") else 1,
form["app"]["name"],
form["app"]["id"],
form.get("module", {"id": -1 if form.get("is_user_registration") else 1000})["id"],
form.get("form", {"id": -1})["id"],
)
if form["has_app"]
else (2, form["xmlns"], form["app"]["id"]),
)
# if there is a custom group export defined grab it here
groups = HQGroupExportConfiguration.by_domain(self.domain)
context = super(ExcelExportReport, self).report_context
context.update(forms=forms, edit=self.request.GET.get("edit") == "true", group_exports=groups)
return context
def _get_daily_saved_export_ids(domain):
group_config = HQGroupExportConfiguration.get_for_domain(domain)
return set(group_config.custom_export_ids)
def emailed_export_groups(self):
"""The groups of saved exports by domain for daily emailed exports.
"""
return HQGroupExportConfiguration.by_domain(self.domain)
def rebuild_export_task(groupexport_id, index, output_dir='couch', last_access_cutoff=None, filter=None):
from couchexport.groupexports import rebuild_export
group_config = HQGroupExportConfiguration.get(groupexport_id)
config, schema = group_config.all_exports[index]
rebuild_export(config, schema, output_dir, last_access_cutoff, filter=filter)
def report_context(self):
# This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids.
# However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}]
forms = []
unknown_forms = []
startkey = [self.domain]
db = Application.get_db() # the view emits from both forms and applications
size_hash = self._get_domain_attachments_size()
for f in db.view('exports_forms/by_xmlns',
startkey=startkey, endkey=startkey + [{}], group=True,
stale=settings.COUCH_STALE_QUERY):
form = f['value']
if form.get('app_deleted') and not form.get('submissions'):
continue
if 'app' in form:
form['has_app'] = True
else:
app_id = f['key'][1] or ''
form['app'] = {
'id': app_id
}
form['has_app'] = False
form['show_xmlns'] = True
unknown_forms.append(form)
form['current_app'] = form.get('app')
if 'id' in form['app']:
key = (form['app']['id'], form['xmlns'])
else:
key = None
if key in size_hash:
form['size'] = size_hash[key]
else:
form['size'] = None
forms.append(form)
if unknown_forms:
apps = db.view('exports_forms/by_xmlns',
startkey=['^Application', self.domain],
endkey=['^Application', self.domain, {}],
reduce=False,
stale=settings.COUCH_STALE_QUERY,
)
possibilities = defaultdict(list)
for app in apps:
# index by xmlns
x = app['value']
x['has_app'] = True
possibilities[app['key'][2]].append(x)
class AppCache(dict):
def __init__(self, domain):
super(AppCache, self).__init__()
self.domain = domain
def __getitem__(self, item):
if not self.has_key(item):
try:
self[item] = get_app(app_id=item, domain=self.domain)
except Http404:
pass
return super(AppCache, self).__getitem__(item)
app_cache = AppCache(self.domain)
for form in unknown_forms:
app = None
if form['app']['id']:
try:
app = app_cache[form['app']['id']]
form['has_app'] = True
except KeyError:
form['app_does_not_exist'] = True
form['possibilities'] = possibilities[form['xmlns']]
if form['possibilities']:
form['duplicate'] = True
else:
if app.domain != self.domain:
logging.error("submission tagged with app from wrong domain: %s" % app.get_id)
else:
if app.copy_of:
try:
app = app_cache[app.copy_of]
form['app_copy'] = {'id': app.get_id, 'name': app.name}
except KeyError:
form['app_copy'] = {'id': app.copy_of, 'name': '?'}
if app.is_deleted():
form['app_deleted'] = {'id': app.get_id}
try:
app_forms = app.get_xmlns_map()[form['xmlns']]
except AttributeError:
# it's a remote app
app_forms = None
if app_forms:
app_form = app_forms[0]
if app_form.doc_type == 'UserRegistrationForm':
form['is_user_registration'] = True
else:
app_module = app_form.get_module()
form['module'] = app_module
form['form'] = app_form
form['show_xmlns'] = False
if not form.get('app_copy') and not form.get('app_deleted'):
form['no_suggestions'] = True
if app:
form['app'] = {'id': app.get_id, 'name': app.name, 'langs': app.langs}
else:
form['possibilities'] = possibilities[form['xmlns']]
if form['possibilities']:
form['duplicate'] = True
else:
form['no_suggestions'] = True
key = (None, form['xmlns'])
form['size'] = size_hash.get(key, None)
def _sortkey(form):
app_id = form['app']['id']
if form['has_app']:
order = 0 if not form.get('app_deleted') else 1
app_name = form['app']['name']
module = form.get('module')
if module:
# module is sometimes wrapped json, sometimes a dict!
module_id = module['id'] if 'id' in module else module.id
else:
module_id = -1 if form.get('is_user_registration') else 1000
app_form = form.get('form')
if app_form:
# app_form is sometimes wrapped json, sometimes a dict!
form_id = app_form['id'] if 'id' in app_form else app_form.id
else:
form_id = -1
return (order, app_name, app_id, module_id, form_id)
else:
form_xmlns = form['xmlns']
return (2, form_xmlns, app_id)
forms = sorted(forms, key=_sortkey)
# if there is a custom group export defined grab it here
groups = HQGroupExportConfiguration.by_domain(self.domain)
context = super(ExcelExportReport, self).report_context
context.update(
forms=forms,
edit=self.request.GET.get('edit') == 'true',
group_exports=[group.form_exports for group in groups
if group.form_exports],
report_slug=self.slug,
property_hash=self.properties(size_hash),
)
return context
def rebuild_export_task(groupexport_id, index, output_dir="couch", last_access_cutoff=None, filter=None):
from couchexport.groupexports import rebuild_export
group_config = HQGroupExportConfiguration.get(groupexport_id)
config, schema = group_config.all_exports[index]
rebuild_export(config, schema, output_dir, last_access_cutoff, filter=filter)
def tearDownClass(cls):
delete_all_docs_by_doc_type(HQGroupExportConfiguration.get_db(),
(HQGroupExportConfiguration.__name__, ))
super(HQGroupExportConfigurationDbAccessorsTest, cls).tearDownClass()
def get_all_hq_group_export_configs():
from corehq.apps.reports.models import HQGroupExportConfiguration
return imap(
HQGroupExportConfiguration.wrap,
get_all_docs_with_doc_types(HQGroupExportConfiguration.get_db(),
('HQGroupExportConfiguration', )))
def setUpClass(cls):
super(HQGroupExportConfigurationDbAccessorsTest, cls).setUpClass()
HQGroupExportConfiguration(domain='domain1').save()
HQGroupExportConfiguration(domain='domain2').save()
HQGroupExportConfiguration(domain='domain2').save()
def saved_exports():
for row in HQGroupExportConfiguration.view("groupexport/by_domain", reduce=False).all():
export_for_group(row["id"], "couch")
def tearDownClass(cls):
delete_all_docs_by_doc_type(HQGroupExportConfiguration.get_db(), (HQGroupExportConfiguration.__name__,))
super(HQGroupExportConfigurationDbAccessorsTest, cls).tearDownClass()
def report_context(self):
# This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids.
# However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}]
forms = []
unknown_forms = []
startkey = [self.domain]
db = Application.get_db() # the view emits from both forms and applications
size_hash = self._get_domain_attachments_size()
for f in db.view('exports_forms/by_xmlns',
startkey=startkey, endkey=startkey + [{}], group=True,
stale=settings.COUCH_STALE_QUERY):
form = f['value']
if form.get('app_deleted') and not form.get('submissions'):
continue
if 'app' in form:
form['has_app'] = True
else:
app_id = f['key'][1] or ''
form['app'] = {
'id': app_id
}
form['has_app'] = False
form['show_xmlns'] = True
unknown_forms.append(form)
form['current_app'] = form.get('app')
if 'id' in form['app']:
key = (form['app']['id'], form['xmlns'])
else:
key = None
if key in size_hash:
form['size'] = size_hash[key]
else:
form['size'] = None
forms.append(form)
if unknown_forms:
apps = db.view('exports_forms/by_xmlns',
startkey=['^Application', self.domain],
endkey=['^Application', self.domain, {}],
reduce=False,
stale=settings.COUCH_STALE_QUERY,
)
possibilities = defaultdict(list)
for app in apps:
# index by xmlns
x = app['value']
x['has_app'] = True
possibilities[app['key'][2]].append(x)
class AppCache(dict):
def __init__(self, domain):
super(AppCache, self).__init__()
self.domain = domain
def __getitem__(self, item):
if not self.has_key(item):
try:
self[item] = get_app(app_id=item, domain=self.domain)
except Http404:
pass
return super(AppCache, self).__getitem__(item)
app_cache = AppCache(self.domain)
for form in unknown_forms:
app = None
if form['app']['id']:
try:
app = app_cache[form['app']['id']]
form['has_app'] = True
except KeyError:
form['app_does_not_exist'] = True
form['possibilities'] = possibilities[form['xmlns']]
if form['possibilities']:
form['duplicate'] = True
else:
if app.domain != self.domain:
logging.error("submission tagged with app from wrong domain: %s" % app.get_id)
else:
if app.copy_of:
try:
app = app_cache[app.copy_of]
form['app_copy'] = {'id': app.get_id, 'name': app.name}
except KeyError:
form['app_copy'] = {'id': app.copy_of, 'name': '?'}
if app.is_deleted():
form['app_deleted'] = {'id': app.get_id}
try:
app_forms = app.get_xmlns_map()[form['xmlns']]
except AttributeError:
# it's a remote app
app_forms = None
if app_forms:
app_form = app_forms[0]
if app_form.doc_type == 'UserRegistrationForm':
form['is_user_registration'] = True
else:
app_module = app_form.get_module()
form['module'] = app_module
form['form'] = app_form
form['show_xmlns'] = False
if not form.get('app_copy') and not form.get('app_deleted'):
form['no_suggestions'] = True
if app:
form['app'] = {'id': app.get_id, 'name': app.name, 'langs': app.langs}
else:
form['possibilities'] = possibilities[form['xmlns']]
if form['possibilities']:
form['duplicate'] = True
else:
form['no_suggestions'] = True
key = (None, form['xmlns'])
form['size'] = size_hash.get(key, None)
def _sortkey(form):
app_id = form['app']['id']
if form['has_app']:
order = 0 if not form.get('app_deleted') else 1
app_name = form['app']['name']
module = form.get('module')
if module:
# module is sometimes wrapped json, sometimes a dict!
module_id = module['id'] if 'id' in module else module.id
else:
module_id = -1 if form.get('is_user_registration') else 1000
app_form = form.get('form')
if app_form:
# app_form is sometimes wrapped json, sometimes a dict!
form_id = app_form['id'] if 'id' in app_form else app_form.id
else:
form_id = -1
return (order, app_name, app_id, module_id, form_id)
else:
form_xmlns = form['xmlns']
return (2, form_xmlns, app_id)
forms = sorted(forms, key=_sortkey)
# if there is a custom group export defined grab it here
groups = HQGroupExportConfiguration.by_domain(self.domain)
context = super(ExcelExportReport, self).report_context
# Check if any custom exports are in the size hash
saved_exports_has_media = any((e.app_id, e.index[1]) in size_hash for e in context['saved_exports'])
context.update(
forms=forms,
edit=self.request.GET.get('edit') == 'true',
group_exports=[group.form_exports for group in groups
if group.form_exports],
group_export_cutoff=datetime.utcnow() - timedelta(days=settings.SAVED_EXPORT_ACCESS_CUTOFF),
report_slug=self.slug,
property_hash=self.properties(size_hash),
exports_has_media=size_hash,
saved_exports_has_media=saved_exports_has_media
)
return context
def tearDownClass(cls):
delete_all_docs_by_doc_type(HQGroupExportConfiguration.get_db(), (HQGroupExportConfiguration.__name__,))
def saved_exports():
for group_config in HQGroupExportConfiguration.view("groupexport/by_domain", reduce=False, include_docs=True).all():
export_for_group_async.delay(group_config, "couch")
def report_context(self):
# This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids.
# However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}]
forms = []
unknown_forms = []
for f in get_db().view('exports_forms/by_xmlns', startkey=[self.domain], endkey=[self.domain, {}], group=True):
form = f['value']
if form.get('app_deleted') and not form.get('submissions'):
continue
if 'app' in form:
form['has_app'] = True
else:
app_id = f['key'][1] or ''
form['app'] = {
'id': app_id
}
form['has_app'] = False
form['show_xmlns'] = True
unknown_forms.append(form)
form['current_app'] = form.get('app')
forms.append(form)
if unknown_forms:
apps = get_db().view('exports_forms/by_xmlns',
startkey=['^Application', self.domain],
endkey=['^Application', self.domain, {}],
reduce=False,
)
possibilities = defaultdict(list)
for app in apps:
# index by xmlns
x = app['value']
x['has_app'] = True
possibilities[app['key'][2]].append(x)
class AppCache(dict):
def __init__(self, domain):
super(AppCache, self).__init__()
self.domain = domain
def __getitem__(self, item):
if not self.has_key(item):
try:
self[item] = get_app(app_id=item, domain=self.domain)
except Http404:
pass
return super(AppCache, self).__getitem__(item)
app_cache = AppCache(self.domain)
for form in unknown_forms:
app = None
if form['app']['id']:
try:
app = app_cache[form['app']['id']]
form['has_app'] = True
except KeyError:
form['app_does_not_exist'] = True
form['possibilities'] = possibilities[form['xmlns']]
if form['possibilities']:
form['duplicate'] = True
else:
if app.domain != self.domain:
logging.error("submission tagged with app from wrong domain: %s" % app.get_id)
else:
if app.copy_of:
try:
app = app_cache[app.copy_of]
form['app_copy'] = {'id': app.get_id, 'name': app.name}
except KeyError:
form['app_copy'] = {'id': app.copy_of, 'name': '?'}
if app.is_deleted():
form['app_deleted'] = {'id': app.get_id}
try:
app_forms = app.get_xmlns_map()[form['xmlns']]
except AttributeError:
# it's a remote app
app_forms = None
if app_forms:
app_form = app_forms[0]
if app_form.doc_type == 'UserRegistrationForm':
form['is_user_registration'] = True
else:
app_module = app_form.get_module()
form['module'] = app_module
form['form'] = app_form
form['show_xmlns'] = False
if not form.get('app_copy') and not form.get('app_deleted'):
form['no_suggestions'] = True
if app:
form['app'] = {'id': app.get_id, 'name': app.name, 'langs': app.langs}
else:
form['possibilities'] = possibilities[form['xmlns']]
if form['possibilities']:
form['duplicate'] = True
else:
form['no_suggestions'] = True
forms = sorted(forms, key=lambda form:\
(0 if not form.get('app_deleted') else 1,
form['app']['name'],
form['app']['id'],
form.get('module', {'id': -1 if form.get('is_user_registration') else 1000})['id'], form.get('form', {'id': -1})['id']
) if form['has_app'] else\
(2, form['xmlns'], form['app']['id'])
)
# if there is a custom group export defined grab it here
groups = HQGroupExportConfiguration.by_domain(self.domain)
context = super(ExcelExportReport, self).report_context
context.update(
forms=forms,
edit=self.request.GET.get('edit') == 'true',
group_exports=groups
)
return context
def report_context(self):
# This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids.
# However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}]
forms = []
unknown_forms = []
startkey = [self.domain]
db = Application.get_db() # the view emits from both forms and applications
size_hash = self._get_domain_attachments_size()
for f in db.view(
"exports_forms/by_xmlns",
startkey=startkey,
endkey=startkey + [{}],
group=True,
stale=settings.COUCH_STALE_QUERY,
):
form = f["value"]
if form.get("app_deleted") and not form.get("submissions"):
continue
if "app" in form:
form["has_app"] = True
else:
app_id = f["key"][1] or ""
form["app"] = {"id": app_id}
form["has_app"] = False
form["show_xmlns"] = True
unknown_forms.append(form)
form["current_app"] = form.get("app")
if "id" in form["app"]:
key = (form["app"]["id"], form["xmlns"])
else:
key = None
if key in size_hash:
form["size"] = size_hash[key]
else:
form["size"] = None
forms.append(form)
if unknown_forms:
apps = db.view(
"exports_forms/by_xmlns",
startkey=["^Application", self.domain],
endkey=["^Application", self.domain, {}],
reduce=False,
stale=settings.COUCH_STALE_QUERY,
)
possibilities = defaultdict(list)
for app in apps:
# index by xmlns
x = app["value"]
x["has_app"] = True
possibilities[app["key"][2]].append(x)
class AppCache(dict):
def __init__(self, domain):
super(AppCache, self).__init__()
self.domain = domain
def __getitem__(self, item):
if not self.has_key(item):
try:
self[item] = get_app(app_id=item, domain=self.domain)
except Http404:
pass
return super(AppCache, self).__getitem__(item)
app_cache = AppCache(self.domain)
for form in unknown_forms:
app = None
if form["app"]["id"]:
try:
app = app_cache[form["app"]["id"]]
form["has_app"] = True
except KeyError:
form["app_does_not_exist"] = True
form["possibilities"] = possibilities[form["xmlns"]]
if form["possibilities"]:
form["duplicate"] = True
else:
if app.domain != self.domain:
logging.error("submission tagged with app from wrong domain: %s" % app.get_id)
else:
if app.copy_of:
try:
app = app_cache[app.copy_of]
form["app_copy"] = {"id": app.get_id, "name": app.name}
except KeyError:
form["app_copy"] = {"id": app.copy_of, "name": "?"}
if app.is_deleted():
form["app_deleted"] = {"id": app.get_id}
try:
app_forms = app.get_xmlns_map()[form["xmlns"]]
except AttributeError:
# it's a remote app
app_forms = None
if app_forms:
app_form = app_forms[0]
if app_form.doc_type == "UserRegistrationForm":
form["is_user_registration"] = True
else:
app_module = app_form.get_module()
form["module"] = app_module
form["form"] = app_form
form["show_xmlns"] = False
if not form.get("app_copy") and not form.get("app_deleted"):
form["no_suggestions"] = True
if app:
form["app"] = {"id": app.get_id, "name": app.name, "langs": app.langs}
else:
form["possibilities"] = possibilities[form["xmlns"]]
if form["possibilities"]:
form["duplicate"] = True
else:
form["no_suggestions"] = True
key = (None, form["xmlns"])
form["size"] = size_hash.get(key, None)
def _sortkey(form):
app_id = form["app"]["id"]
if form["has_app"]:
order = 0 if not form.get("app_deleted") else 1
app_name = form["app"]["name"]
module = form.get("module")
if module:
# module is sometimes wrapped json, sometimes a dict!
module_id = module["id"] if "id" in module else module.id
else:
module_id = -1 if form.get("is_user_registration") else 1000
app_form = form.get("form")
if app_form:
# app_form is sometimes wrapped json, sometimes a dict!
form_id = app_form["id"] if "id" in app_form else app_form.id
else:
form_id = -1
return (order, app_name, app_id, module_id, form_id)
else:
form_xmlns = form["xmlns"]
return (2, form_xmlns, app_id)
forms = sorted(forms, key=_sortkey)
# if there is a custom group export defined grab it here
groups = HQGroupExportConfiguration.by_domain(self.domain)
context = super(ExcelExportReport, self).report_context
# Check if any custom exports are in the size hash
saved_exports_has_media = any((e.app_id, e.index[1]) in size_hash for e in context["saved_exports"])
context.update(
forms=forms,
edit=self.request.GET.get("edit") == "true",
group_exports=[group.form_exports for group in groups if group.form_exports],
group_export_cutoff=datetime.utcnow() - timedelta(days=settings.SAVED_EXPORT_ACCESS_CUTOFF),
report_slug=self.slug,
property_hash=self.properties(size_hash),
exports_has_media=size_hash,
saved_exports_has_media=saved_exports_has_media,
)
return context
def export_for_group_async(group_config_id):
# exclude exports not accessed within the last 7 days
last_access_cutoff = datetime.utcnow() - timedelta(
days=settings.SAVED_EXPORT_ACCESS_CUTOFF)
group_config = HQGroupExportConfiguration.get(group_config_id)
export_for_group(group_config, last_access_cutoff=last_access_cutoff)
def _get_daily_saved_export_ids(domain):
group_config = HQGroupExportConfiguration.get_for_domain(domain)
return set(group_config.custom_export_ids)
def saved_exports():
for group_config in HQGroupExportConfiguration.view(
"groupexport/by_domain", reduce=False, include_docs=True).all():
export_for_group(group_config, "couch")
