def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ post_data = self.post_data custom_export_json = post_data['custom_export'] if post_data['presave'] and not self.allow_daily_saved: raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports")) if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard: raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard")) if custom_export_json["is_safe"] and not self.allow_deid: raise BadExportConfiguration(_("This user does not have permission to create a de-identified export")) SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates') for key in SAFE_KEYS: self.custom_export[key] = custom_export_json[key] # update the custom export index (to stay in sync) schema_id = self.custom_export.schema_id schema = ExportSchema.get(schema_id) self.custom_export.index = schema.index self.presave = post_data['presave'] self.export_stock = post_data['export_stock'] self.custom_export.tables = [ ExportTable.wrap(table) for table in custom_export_json['tables'] ] table_dict = dict((t.index, t) for t in self.custom_export.tables) for table in self.custom_export.tables: if table.index in table_dict: table_dict[table.index].columns = table.columns else: self.custom_export.tables.append( ExportTable( index=table.index, display=self.custom_export.name, columns=table.columns ) ) self.update_custom_params() self.custom_export.custom_validate() self.custom_export.save() touch_exports(self.domain) if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id) return self.custom_export.get_id
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ post_data = self.post_data custom_export_json = post_data['custom_export'] if post_data['presave'] and not self.allow_daily_saved: raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports")) if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard: raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard")) if custom_export_json["is_safe"] and not self.allow_deid: raise BadExportConfiguration(_("This user does not have permission to create a de-identified export")) SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates') for key in SAFE_KEYS: self.custom_export[key] = custom_export_json[key] # update the custom export index (to stay in sync) schema_id = self.custom_export.schema_id schema = ExportSchema.get(schema_id) self.custom_export.index = schema.index self.presave = post_data['presave'] self.export_stock = post_data['export_stock'] self.custom_export.tables = [ ExportTable.wrap(table) for table in custom_export_json['tables'] ] table_dict = dict((t.index, t) for t in self.custom_export.tables) for table in self.custom_export.tables: if table.index in table_dict: table_dict[table.index].columns = table.columns else: self.custom_export.tables.append( ExportTable( index=table.index, display=self.custom_export.name, columns=table.columns ) ) self.update_custom_params() self.custom_export.custom_validate() self.custom_export.save() touch_exports(self.domain) if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id) return self.custom_export.get_id
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ schema = ExportSchema.get(self.request.POST["schema"]) self.custom_export.index = schema.index self.custom_export.schema_id = self.request.POST["schema"] self.custom_export.name = self.request.POST["name"] self.custom_export.default_format = self.request.POST["format"] or Format.XLS_2007 self.custom_export.is_safe = bool(self.request.POST.get('is_safe')) self.presave = bool(self.request.POST.get('presave')) table = self.request.POST["table"] cols = self.request.POST['order'].strip().split() @list @inline def export_cols(): for col in cols: transform = self.request.POST.get('%s transform' % col) or None if transform: transform = SerializableFunction.loads(transform) yield ExportColumn( index=col, display=self.request.POST["%s display" % col], transform=transform ) export_table = ExportTable(index=table, display=self.request.POST["name"], columns=export_cols) self.custom_export.tables = [export_table] self.custom_export.order = cols table_dict = dict([t.index, t] for t in self.custom_export.tables) if table in table_dict: table_dict[table].columns = export_cols else: self.custom_export.tables.append(ExportTable(index=table, display=self.custom_export.name, columns=export_cols)) if self.export_type == 'form': self.custom_export.include_errors = bool(self.request.POST.get("include-errors")) self.custom_export.app_id = self.request.POST.get('app_id') self.custom_export.save() if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ post_data = self.post_data custom_export_json = post_data['custom_export'] SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates') for key in SAFE_KEYS: self.custom_export[key] = custom_export_json[key] # update the custom export index (to stay in sync) schema_id = self.custom_export.schema_id schema = ExportSchema.get(schema_id) self.custom_export.index = schema.index self.presave = post_data['presave'] self.export_stock = post_data['export_stock'] self.custom_export.tables = [ ExportTable.wrap(table) for table in custom_export_json['tables'] ] table_dict = dict((t.index, t) for t in self.custom_export.tables) for table in self.custom_export.tables: if table.index in table_dict: table_dict[table.index].columns = table.columns else: self.custom_export.tables.append( ExportTable( index=table.index, display=self.custom_export.name, columns=table.columns ) ) self.update_custom_params() self.custom_export.custom_validate() self.custom_export.save() touch_exports(self.domain) if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id) return self.custom_export.get_id
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ post_data = self.post_data custom_export_json = post_data['custom_export'] SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates') for key in SAFE_KEYS: self.custom_export[key] = custom_export_json[key] # update the custom export index (to stay in sync) schema_id = self.custom_export.schema_id schema = ExportSchema.get(schema_id) self.custom_export.index = schema.index self.presave = post_data['presave'] self.export_stock = post_data['export_stock'] self.custom_export.tables = [ ExportTable.wrap(table) for table in custom_export_json['tables'] ] table_dict = dict((t.index, t) for t in self.custom_export.tables) for table in self.custom_export.tables: if table.index in table_dict: table_dict[table.index].columns = table.columns else: self.custom_export.tables.append( ExportTable( index=table.index, display=self.custom_export.name, columns=table.columns ) ) self.update_custom_params() self.custom_export.custom_validate() self.custom_export.save() touch_exports(self.domain) if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id) return self.custom_export.get_id
def __init__(self, request, domain, export_id=None, minimal=False): self.request = request self.domain = domain self.presave = False self.transform_dates = False self.creating_new_export = not bool(export_id) self.minimal = minimal if export_id: self.custom_export = self.ExportSchemaClass.get(export_id) # also update the schema to include potential new stuff self.custom_export.update_schema() # enable configuring saved exports from this page saved_group = HQGroupExportConfiguration.get_for_domain( self.domain) self.presave = export_id in saved_group.custom_export_ids self.export_stock = self.has_stock_column() try: assert self.custom_export.doc_type == 'SavedExportSchema', 'bad export doc type' assert self.custom_export.type == self.export_type, 'wrong export type specified' assert self.custom_export.index[ 0] == domain, 'bad export doc domain' except AssertionError as e: raise BadExportConfiguration(str(e)) else: self.custom_export = self.ExportSchemaClass(type=self.export_type) self.export_stock = False
def __init__(self, request, domain, export_id=None): self.request = request self.domain = domain self.export_type = request.GET.get('type', 'form') self.presave = False if self.export_type == 'form': self.ExportSchemaClass = FormExportSchema else: self.ExportSchemaClass = SavedExportSchema if export_id: self.custom_export = self.ExportSchemaClass.get(export_id) # also update the schema to include potential new stuff self.custom_export.update_schema() # enable configuring saved exports from this page saved_group = HQGroupExportConfiguration.get_for_domain(self.domain) self.presave = export_id in saved_group.custom_export_ids assert(self.custom_export.doc_type == 'SavedExportSchema') assert(self.custom_export.type == self.export_type) assert(self.custom_export.index[0] == domain) else: self.custom_export = self.ExportSchemaClass(type=self.export_type) if self.export_type == 'form': self.custom_export.app_id = request.GET.get('app_id')
def __init__(self, request, domain, export_id=None, minimal=False): self.request = request self.domain = domain self.presave = False self.transform_dates = False self.creating_new_export = not bool(export_id) self.minimal = minimal if export_id: self.custom_export = self.ExportSchemaClass.get(export_id) # also update the schema to include potential new stuff self.custom_export.update_schema() # enable configuring saved exports from this page saved_group = HQGroupExportConfiguration.get_for_domain(self.domain) self.presave = export_id in saved_group.custom_export_ids self.export_stock = self.has_stock_column() try: assert self.custom_export.doc_type == 'SavedExportSchema', 'bad export doc type' assert self.custom_export.type == self.export_type, 'wrong export type specified' assert self.custom_export.index[0] == domain, 'bad export doc domain' except AssertionError, e: raise BadExportConfiguration(str(e))
def hq_update_saved_export(req, domain): group_id = req.POST["group_export_id"] index = int(req.POST["index"]) group_config = HQGroupExportConfiguration.get(group_id) assert domain == group_config.domain config, schema = group_config.all_exports[index] rebuild_export(config, schema, "couch") messages.success(req, _("The data for {} has been refreshed!").format(config.name)) return HttpResponseRedirect(reverse(DataInterfaceDispatcher.name(), args=[domain, req.POST["report_slug"]]))
def get_all_hq_group_export_configs(): from corehq.apps.reports.models import HQGroupExportConfiguration return imap( HQGroupExportConfiguration.wrap, get_all_docs_with_doc_types( HQGroupExportConfiguration.get_db(), ('HQGroupExportConfiguration',) ) )
def report_context(self): context = super(CaseExportReport, self).report_context case_types = get_case_types_for_domain(self.domain) groups = HQGroupExportConfiguration.by_domain(self.domain) context.update( case_types=case_types, group_exports=[group.case_exports for group in groups if group.case_exports], report_slug=self.slug, ) context["case_format"] = self.request.GET.get("case_format") or "csv" return context
def report_context(self): context = super(CaseExportReport, self).report_context case_types = get_case_types_for_domain(self.domain) groups = HQGroupExportConfiguration.by_domain(self.domain) context.update( case_types=case_types, group_exports=[group.case_exports for group in groups if group.case_exports], report_slug=self.slug, ) context['case_format'] = self.request.GET.get('case_format') or 'csv' return context
def hq_update_saved_export(req, domain): group_id = req.POST['group_export_id'] index = int(req.POST['index']) group_config = HQGroupExportConfiguration.get(group_id) assert domain == group_config.domain config, schema = group_config.all_exports[index] rebuild_export(config, schema, 'couch') messages.success( req, _('The data for {} has been refreshed!').format(config.name)) return HttpResponseRedirect( reverse(DataInterfaceDispatcher.name(), args=[domain, req.POST['report_slug']]))
def report_context(self): context = super(CaseExportReport, self).report_context cases = get_db().view("hqcase/types_by_domain", startkey=[self.domain], endkey=[self.domain, {}], reduce=True, group=True, group_level=2).all() groups = HQGroupExportConfiguration.by_domain(self.domain) context.update( case_types=[case['key'][1] for case in cases], group_exports=[group.case_exports for group in groups if group.case_exports], ) return context
def report_context(self): context = super(CaseExportReport, self).report_context cases = get_db().view("hqcase/types_by_domain", startkey=[self.domain], endkey=[self.domain, {}], reduce=True, group=True, group_level=2).all() groups = HQGroupExportConfiguration.by_domain(self.domain) context.update( case_types=[case['key'][1] for case in cases], group_exports=[ group.case_exports for group in groups if group.case_exports ], ) context['case_format'] = self.request.GET.get('case_format') or 'csv' return context
def report_context(self): context = super(CaseExportReport, self).report_context cases = CommCareCase.get_db().view("hqcase/types_by_domain", startkey=[self.domain], endkey=[self.domain, {}], reduce=True, group=True, group_level=2).all() groups = HQGroupExportConfiguration.by_domain(self.domain) context.update( case_types=[case['key'][1] for case in cases], group_exports=[group.case_exports for group in groups if group.case_exports], report_slug=self.slug, ) context['case_format'] = self.request.GET.get('case_format') or 'csv' return context
def __init__(self, request, domain, export_id=None): self.request = request self.domain = domain self.export_type = request.GET.get('type', 'form') self.presave = False if export_id: self.custom_export = self.ExportSchemaClass.get(export_id) # also update the schema to include potential new stuff self.custom_export.update_schema() # enable configuring saved exports from this page saved_group = HQGroupExportConfiguration.get_for_domain(self.domain) self.presave = export_id in saved_group.custom_export_ids assert(self.custom_export.doc_type == 'SavedExportSchema') assert(self.custom_export.type == self.export_type) assert(self.custom_export.index[0] == domain) else: self.custom_export = self.ExportSchemaClass(type=self.export_type)
def __init__(self, request, domain, export_id=None): self.request = request self.domain = domain self.presave = False self.transform_dates = False self.creating_new_export = not bool(export_id) if export_id: self.custom_export = self.ExportSchemaClass.get(export_id) # also update the schema to include potential new stuff self.custom_export.update_schema() # enable configuring saved exports from this page saved_group = HQGroupExportConfiguration.get_for_domain(self.domain) self.presave = export_id in saved_group.custom_export_ids assert(self.custom_export.doc_type == 'SavedExportSchema') assert(self.custom_export.type == self.export_type) assert(self.custom_export.index[0] == domain) else: self.custom_export = self.ExportSchemaClass(type=self.export_type)
def test_daily_saved_conversion(self, _): # ID is from corehq/apps/export/tests/data/saved_export_schemas/case.json self.group_config = HQGroupExportConfiguration.add_custom_export( self.domain, '92e5f9a6624a637c2080957475cd446d') self.group_config.save() self.addCleanup(self.group_config.delete) instance, _ = self._convert_case_export('case') self.assertEqual(instance.transform_dates, True) self.assertEqual(instance.name, 'Case Example') self.assertEqual(instance.export_format, 'csv') self.assertEqual(instance.is_deidentified, False) self.assertEqual(instance.is_daily_saved_export, True) table = instance.get_table(MAIN_TABLE) self.assertEqual(table.label, 'Cases') self.assertTrue(table.selected) index, column = table.get_column([PathNode(name='DOB')], 'ExportItem', None) self.assertEqual(column.label, 'DOB Saved') self.assertEqual(column.selected, True)
def test_daily_saved_conversion(self, _): # ID is from corehq/apps/export/tests/data/saved_export_schemas/case.json self.group_config = HQGroupExportConfiguration.add_custom_export( self.domain, "92e5f9a6624a637c2080957475cd446d" ) self.group_config.save() self.addCleanup(self.group_config.delete) instance, _ = self._convert_case_export("case") self.assertEqual(instance.transform_dates, True) self.assertEqual(instance.name, "Case Example") self.assertEqual(instance.export_format, "csv") self.assertEqual(instance.is_deidentified, False) self.assertEqual(instance.is_daily_saved_export, True) table = instance.get_table(MAIN_TABLE) self.assertEqual(table.label, "Cases") self.assertTrue(table.selected) index, column = table.get_column([PathNode(name="DOB")], "ExportItem", None) self.assertEqual(column.label, "DOB Saved") self.assertEqual(column.selected, True)
def saved_exports(): for row in HQGroupExportConfiguration.view("groupexport/by_domain", reduce=False).all(): export_for_group(row["id"], "couch")
def report_context(self): # This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids. # However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}] forms = [] unknown_forms = [] for f in get_db().view("reports/forms_by_xmlns", startkey=[self.domain], endkey=[self.domain, {}], group=True): form = f["value"] if form.get("app_deleted") and not form.get("submissions"): continue if "app" in form: form["has_app"] = True else: app_id = f["key"][1] or "" form["app"] = {"id": app_id} form["has_app"] = False form["show_xmlns"] = True unknown_forms.append(form) form["current_app"] = form.get("app") forms.append(form) if unknown_forms: apps = get_db().view( "reports/forms_by_xmlns", startkey=["^Application", self.domain], endkey=["^Application", self.domain, {}], reduce=False, ) possibilities = defaultdict(list) for app in apps: # index by xmlns x = app["value"] x["has_app"] = True possibilities[app["key"][2]].append(x) class AppCache(dict): def __init__(self, domain): super(AppCache, self).__init__() self.domain = domain def __getitem__(self, item): if not self.has_key(item): try: self[item] = get_app(app_id=item, domain=self.domain) except Http404: pass return super(AppCache, self).__getitem__(item) app_cache = AppCache(self.domain) for form in unknown_forms: app = None if form["app"]["id"]: try: app = app_cache[form["app"]["id"]] form["has_app"] = True except KeyError: form["app_does_not_exist"] = True form["possibilities"] = possibilities[form["xmlns"]] if form["possibilities"]: form["duplicate"] = True else: if app.domain != self.domain: logging.error("submission tagged with app from wrong domain: %s" % app.get_id) else: if app.copy_of: try: app = app_cache[app.copy_of] form["app_copy"] = {"id": app.get_id, "name": app.name} except KeyError: form["app_copy"] = {"id": app.copy_of, "name": "?"} if app.is_deleted(): form["app_deleted"] = {"id": app.get_id} try: app_forms = app.get_xmlns_map()[form["xmlns"]] except AttributeError: # it's a remote app app_forms = None if app_forms: app_form = app_forms[0] if app_form.doc_type == "UserRegistrationForm": form["is_user_registration"] = True else: app_module = app_form.get_module() form["module"] = app_module form["form"] = app_form form["show_xmlns"] = False if not form.get("app_copy") and not form.get("app_deleted"): form["no_suggestions"] = True if app: form["app"] = {"id": app.get_id, "name": app.name, "langs": app.langs} else: form["possibilities"] = possibilities[form["xmlns"]] if form["possibilities"]: form["duplicate"] = True else: form["no_suggestions"] = True forms = sorted( forms, key=lambda form: ( 0 if not form.get("app_deleted") else 1, form["app"]["name"], form["app"]["id"], form.get("module", {"id": -1 if form.get("is_user_registration") else 1000})["id"], form.get("form", {"id": -1})["id"], ) if form["has_app"] else (2, form["xmlns"], form["app"]["id"]), ) # if there is a custom group export defined grab it here groups = HQGroupExportConfiguration.by_domain(self.domain) context = super(ExcelExportReport, self).report_context context.update(forms=forms, edit=self.request.GET.get("edit") == "true", group_exports=groups) return context
def _get_daily_saved_export_ids(domain): group_config = HQGroupExportConfiguration.get_for_domain(domain) return set(group_config.custom_export_ids)
def emailed_export_groups(self): """The groups of saved exports by domain for daily emailed exports. """ return HQGroupExportConfiguration.by_domain(self.domain)
def rebuild_export_task(groupexport_id, index, output_dir='couch', last_access_cutoff=None, filter=None): from couchexport.groupexports import rebuild_export group_config = HQGroupExportConfiguration.get(groupexport_id) config, schema = group_config.all_exports[index] rebuild_export(config, schema, output_dir, last_access_cutoff, filter=filter)
def report_context(self): # This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids. # However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}] forms = [] unknown_forms = [] startkey = [self.domain] db = Application.get_db() # the view emits from both forms and applications size_hash = self._get_domain_attachments_size() for f in db.view('exports_forms/by_xmlns', startkey=startkey, endkey=startkey + [{}], group=True, stale=settings.COUCH_STALE_QUERY): form = f['value'] if form.get('app_deleted') and not form.get('submissions'): continue if 'app' in form: form['has_app'] = True else: app_id = f['key'][1] or '' form['app'] = { 'id': app_id } form['has_app'] = False form['show_xmlns'] = True unknown_forms.append(form) form['current_app'] = form.get('app') if 'id' in form['app']: key = (form['app']['id'], form['xmlns']) else: key = None if key in size_hash: form['size'] = size_hash[key] else: form['size'] = None forms.append(form) if unknown_forms: apps = db.view('exports_forms/by_xmlns', startkey=['^Application', self.domain], endkey=['^Application', self.domain, {}], reduce=False, stale=settings.COUCH_STALE_QUERY, ) possibilities = defaultdict(list) for app in apps: # index by xmlns x = app['value'] x['has_app'] = True possibilities[app['key'][2]].append(x) class AppCache(dict): def __init__(self, domain): super(AppCache, self).__init__() self.domain = domain def __getitem__(self, item): if not self.has_key(item): try: self[item] = get_app(app_id=item, domain=self.domain) except Http404: pass return super(AppCache, self).__getitem__(item) app_cache = AppCache(self.domain) for form in unknown_forms: app = None if form['app']['id']: try: app = app_cache[form['app']['id']] form['has_app'] = True except KeyError: form['app_does_not_exist'] = True form['possibilities'] = possibilities[form['xmlns']] if form['possibilities']: form['duplicate'] = True else: if app.domain != self.domain: logging.error("submission tagged with app from wrong domain: %s" % app.get_id) else: if app.copy_of: try: app = app_cache[app.copy_of] form['app_copy'] = {'id': app.get_id, 'name': app.name} except KeyError: form['app_copy'] = {'id': app.copy_of, 'name': '?'} if app.is_deleted(): form['app_deleted'] = {'id': app.get_id} try: app_forms = app.get_xmlns_map()[form['xmlns']] except AttributeError: # it's a remote app app_forms = None if app_forms: app_form = app_forms[0] if app_form.doc_type == 'UserRegistrationForm': form['is_user_registration'] = True else: app_module = app_form.get_module() form['module'] = app_module form['form'] = app_form form['show_xmlns'] = False if not form.get('app_copy') and not form.get('app_deleted'): form['no_suggestions'] = True if app: form['app'] = {'id': app.get_id, 'name': app.name, 'langs': app.langs} else: form['possibilities'] = possibilities[form['xmlns']] if form['possibilities']: form['duplicate'] = True else: form['no_suggestions'] = True key = (None, form['xmlns']) form['size'] = size_hash.get(key, None) def _sortkey(form): app_id = form['app']['id'] if form['has_app']: order = 0 if not form.get('app_deleted') else 1 app_name = form['app']['name'] module = form.get('module') if module: # module is sometimes wrapped json, sometimes a dict! module_id = module['id'] if 'id' in module else module.id else: module_id = -1 if form.get('is_user_registration') else 1000 app_form = form.get('form') if app_form: # app_form is sometimes wrapped json, sometimes a dict! form_id = app_form['id'] if 'id' in app_form else app_form.id else: form_id = -1 return (order, app_name, app_id, module_id, form_id) else: form_xmlns = form['xmlns'] return (2, form_xmlns, app_id) forms = sorted(forms, key=_sortkey) # if there is a custom group export defined grab it here groups = HQGroupExportConfiguration.by_domain(self.domain) context = super(ExcelExportReport, self).report_context context.update( forms=forms, edit=self.request.GET.get('edit') == 'true', group_exports=[group.form_exports for group in groups if group.form_exports], report_slug=self.slug, property_hash=self.properties(size_hash), ) return context
def rebuild_export_task(groupexport_id, index, output_dir="couch", last_access_cutoff=None, filter=None): from couchexport.groupexports import rebuild_export group_config = HQGroupExportConfiguration.get(groupexport_id) config, schema = group_config.all_exports[index] rebuild_export(config, schema, output_dir, last_access_cutoff, filter=filter)
def tearDownClass(cls): delete_all_docs_by_doc_type(HQGroupExportConfiguration.get_db(), (HQGroupExportConfiguration.__name__, )) super(HQGroupExportConfigurationDbAccessorsTest, cls).tearDownClass()
def get_all_hq_group_export_configs(): from corehq.apps.reports.models import HQGroupExportConfiguration return imap( HQGroupExportConfiguration.wrap, get_all_docs_with_doc_types(HQGroupExportConfiguration.get_db(), ('HQGroupExportConfiguration', )))
def setUpClass(cls): super(HQGroupExportConfigurationDbAccessorsTest, cls).setUpClass() HQGroupExportConfiguration(domain='domain1').save() HQGroupExportConfiguration(domain='domain2').save() HQGroupExportConfiguration(domain='domain2').save()
def saved_exports(): for row in HQGroupExportConfiguration.view("groupexport/by_domain", reduce=False).all(): export_for_group(row["id"], "couch")
def tearDownClass(cls): delete_all_docs_by_doc_type(HQGroupExportConfiguration.get_db(), (HQGroupExportConfiguration.__name__,)) super(HQGroupExportConfigurationDbAccessorsTest, cls).tearDownClass()
def report_context(self): # This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids. # However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}] forms = [] unknown_forms = [] startkey = [self.domain] db = Application.get_db() # the view emits from both forms and applications size_hash = self._get_domain_attachments_size() for f in db.view('exports_forms/by_xmlns', startkey=startkey, endkey=startkey + [{}], group=True, stale=settings.COUCH_STALE_QUERY): form = f['value'] if form.get('app_deleted') and not form.get('submissions'): continue if 'app' in form: form['has_app'] = True else: app_id = f['key'][1] or '' form['app'] = { 'id': app_id } form['has_app'] = False form['show_xmlns'] = True unknown_forms.append(form) form['current_app'] = form.get('app') if 'id' in form['app']: key = (form['app']['id'], form['xmlns']) else: key = None if key in size_hash: form['size'] = size_hash[key] else: form['size'] = None forms.append(form) if unknown_forms: apps = db.view('exports_forms/by_xmlns', startkey=['^Application', self.domain], endkey=['^Application', self.domain, {}], reduce=False, stale=settings.COUCH_STALE_QUERY, ) possibilities = defaultdict(list) for app in apps: # index by xmlns x = app['value'] x['has_app'] = True possibilities[app['key'][2]].append(x) class AppCache(dict): def __init__(self, domain): super(AppCache, self).__init__() self.domain = domain def __getitem__(self, item): if not self.has_key(item): try: self[item] = get_app(app_id=item, domain=self.domain) except Http404: pass return super(AppCache, self).__getitem__(item) app_cache = AppCache(self.domain) for form in unknown_forms: app = None if form['app']['id']: try: app = app_cache[form['app']['id']] form['has_app'] = True except KeyError: form['app_does_not_exist'] = True form['possibilities'] = possibilities[form['xmlns']] if form['possibilities']: form['duplicate'] = True else: if app.domain != self.domain: logging.error("submission tagged with app from wrong domain: %s" % app.get_id) else: if app.copy_of: try: app = app_cache[app.copy_of] form['app_copy'] = {'id': app.get_id, 'name': app.name} except KeyError: form['app_copy'] = {'id': app.copy_of, 'name': '?'} if app.is_deleted(): form['app_deleted'] = {'id': app.get_id} try: app_forms = app.get_xmlns_map()[form['xmlns']] except AttributeError: # it's a remote app app_forms = None if app_forms: app_form = app_forms[0] if app_form.doc_type == 'UserRegistrationForm': form['is_user_registration'] = True else: app_module = app_form.get_module() form['module'] = app_module form['form'] = app_form form['show_xmlns'] = False if not form.get('app_copy') and not form.get('app_deleted'): form['no_suggestions'] = True if app: form['app'] = {'id': app.get_id, 'name': app.name, 'langs': app.langs} else: form['possibilities'] = possibilities[form['xmlns']] if form['possibilities']: form['duplicate'] = True else: form['no_suggestions'] = True key = (None, form['xmlns']) form['size'] = size_hash.get(key, None) def _sortkey(form): app_id = form['app']['id'] if form['has_app']: order = 0 if not form.get('app_deleted') else 1 app_name = form['app']['name'] module = form.get('module') if module: # module is sometimes wrapped json, sometimes a dict! module_id = module['id'] if 'id' in module else module.id else: module_id = -1 if form.get('is_user_registration') else 1000 app_form = form.get('form') if app_form: # app_form is sometimes wrapped json, sometimes a dict! form_id = app_form['id'] if 'id' in app_form else app_form.id else: form_id = -1 return (order, app_name, app_id, module_id, form_id) else: form_xmlns = form['xmlns'] return (2, form_xmlns, app_id) forms = sorted(forms, key=_sortkey) # if there is a custom group export defined grab it here groups = HQGroupExportConfiguration.by_domain(self.domain) context = super(ExcelExportReport, self).report_context # Check if any custom exports are in the size hash saved_exports_has_media = any((e.app_id, e.index[1]) in size_hash for e in context['saved_exports']) context.update( forms=forms, edit=self.request.GET.get('edit') == 'true', group_exports=[group.form_exports for group in groups if group.form_exports], group_export_cutoff=datetime.utcnow() - timedelta(days=settings.SAVED_EXPORT_ACCESS_CUTOFF), report_slug=self.slug, property_hash=self.properties(size_hash), exports_has_media=size_hash, saved_exports_has_media=saved_exports_has_media ) return context
def tearDownClass(cls): delete_all_docs_by_doc_type(HQGroupExportConfiguration.get_db(), (HQGroupExportConfiguration.__name__,))
def saved_exports(): for group_config in HQGroupExportConfiguration.view("groupexport/by_domain", reduce=False, include_docs=True).all(): export_for_group_async.delay(group_config, "couch")
def report_context(self): # This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids. # However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}] forms = [] unknown_forms = [] for f in get_db().view('exports_forms/by_xmlns', startkey=[self.domain], endkey=[self.domain, {}], group=True): form = f['value'] if form.get('app_deleted') and not form.get('submissions'): continue if 'app' in form: form['has_app'] = True else: app_id = f['key'][1] or '' form['app'] = { 'id': app_id } form['has_app'] = False form['show_xmlns'] = True unknown_forms.append(form) form['current_app'] = form.get('app') forms.append(form) if unknown_forms: apps = get_db().view('exports_forms/by_xmlns', startkey=['^Application', self.domain], endkey=['^Application', self.domain, {}], reduce=False, ) possibilities = defaultdict(list) for app in apps: # index by xmlns x = app['value'] x['has_app'] = True possibilities[app['key'][2]].append(x) class AppCache(dict): def __init__(self, domain): super(AppCache, self).__init__() self.domain = domain def __getitem__(self, item): if not self.has_key(item): try: self[item] = get_app(app_id=item, domain=self.domain) except Http404: pass return super(AppCache, self).__getitem__(item) app_cache = AppCache(self.domain) for form in unknown_forms: app = None if form['app']['id']: try: app = app_cache[form['app']['id']] form['has_app'] = True except KeyError: form['app_does_not_exist'] = True form['possibilities'] = possibilities[form['xmlns']] if form['possibilities']: form['duplicate'] = True else: if app.domain != self.domain: logging.error("submission tagged with app from wrong domain: %s" % app.get_id) else: if app.copy_of: try: app = app_cache[app.copy_of] form['app_copy'] = {'id': app.get_id, 'name': app.name} except KeyError: form['app_copy'] = {'id': app.copy_of, 'name': '?'} if app.is_deleted(): form['app_deleted'] = {'id': app.get_id} try: app_forms = app.get_xmlns_map()[form['xmlns']] except AttributeError: # it's a remote app app_forms = None if app_forms: app_form = app_forms[0] if app_form.doc_type == 'UserRegistrationForm': form['is_user_registration'] = True else: app_module = app_form.get_module() form['module'] = app_module form['form'] = app_form form['show_xmlns'] = False if not form.get('app_copy') and not form.get('app_deleted'): form['no_suggestions'] = True if app: form['app'] = {'id': app.get_id, 'name': app.name, 'langs': app.langs} else: form['possibilities'] = possibilities[form['xmlns']] if form['possibilities']: form['duplicate'] = True else: form['no_suggestions'] = True forms = sorted(forms, key=lambda form:\ (0 if not form.get('app_deleted') else 1, form['app']['name'], form['app']['id'], form.get('module', {'id': -1 if form.get('is_user_registration') else 1000})['id'], form.get('form', {'id': -1})['id'] ) if form['has_app'] else\ (2, form['xmlns'], form['app']['id']) ) # if there is a custom group export defined grab it here groups = HQGroupExportConfiguration.by_domain(self.domain) context = super(ExcelExportReport, self).report_context context.update( forms=forms, edit=self.request.GET.get('edit') == 'true', group_exports=groups ) return context
def report_context(self): # This map for this view emits twice, once with app_id and once with {}, letting you join across all app_ids. # However, we want to separate out by (app_id, xmlns) pair not just xmlns so we use [domain] to [domain, {}] forms = [] unknown_forms = [] startkey = [self.domain] db = Application.get_db() # the view emits from both forms and applications size_hash = self._get_domain_attachments_size() for f in db.view( "exports_forms/by_xmlns", startkey=startkey, endkey=startkey + [{}], group=True, stale=settings.COUCH_STALE_QUERY, ): form = f["value"] if form.get("app_deleted") and not form.get("submissions"): continue if "app" in form: form["has_app"] = True else: app_id = f["key"][1] or "" form["app"] = {"id": app_id} form["has_app"] = False form["show_xmlns"] = True unknown_forms.append(form) form["current_app"] = form.get("app") if "id" in form["app"]: key = (form["app"]["id"], form["xmlns"]) else: key = None if key in size_hash: form["size"] = size_hash[key] else: form["size"] = None forms.append(form) if unknown_forms: apps = db.view( "exports_forms/by_xmlns", startkey=["^Application", self.domain], endkey=["^Application", self.domain, {}], reduce=False, stale=settings.COUCH_STALE_QUERY, ) possibilities = defaultdict(list) for app in apps: # index by xmlns x = app["value"] x["has_app"] = True possibilities[app["key"][2]].append(x) class AppCache(dict): def __init__(self, domain): super(AppCache, self).__init__() self.domain = domain def __getitem__(self, item): if not self.has_key(item): try: self[item] = get_app(app_id=item, domain=self.domain) except Http404: pass return super(AppCache, self).__getitem__(item) app_cache = AppCache(self.domain) for form in unknown_forms: app = None if form["app"]["id"]: try: app = app_cache[form["app"]["id"]] form["has_app"] = True except KeyError: form["app_does_not_exist"] = True form["possibilities"] = possibilities[form["xmlns"]] if form["possibilities"]: form["duplicate"] = True else: if app.domain != self.domain: logging.error("submission tagged with app from wrong domain: %s" % app.get_id) else: if app.copy_of: try: app = app_cache[app.copy_of] form["app_copy"] = {"id": app.get_id, "name": app.name} except KeyError: form["app_copy"] = {"id": app.copy_of, "name": "?"} if app.is_deleted(): form["app_deleted"] = {"id": app.get_id} try: app_forms = app.get_xmlns_map()[form["xmlns"]] except AttributeError: # it's a remote app app_forms = None if app_forms: app_form = app_forms[0] if app_form.doc_type == "UserRegistrationForm": form["is_user_registration"] = True else: app_module = app_form.get_module() form["module"] = app_module form["form"] = app_form form["show_xmlns"] = False if not form.get("app_copy") and not form.get("app_deleted"): form["no_suggestions"] = True if app: form["app"] = {"id": app.get_id, "name": app.name, "langs": app.langs} else: form["possibilities"] = possibilities[form["xmlns"]] if form["possibilities"]: form["duplicate"] = True else: form["no_suggestions"] = True key = (None, form["xmlns"]) form["size"] = size_hash.get(key, None) def _sortkey(form): app_id = form["app"]["id"] if form["has_app"]: order = 0 if not form.get("app_deleted") else 1 app_name = form["app"]["name"] module = form.get("module") if module: # module is sometimes wrapped json, sometimes a dict! module_id = module["id"] if "id" in module else module.id else: module_id = -1 if form.get("is_user_registration") else 1000 app_form = form.get("form") if app_form: # app_form is sometimes wrapped json, sometimes a dict! form_id = app_form["id"] if "id" in app_form else app_form.id else: form_id = -1 return (order, app_name, app_id, module_id, form_id) else: form_xmlns = form["xmlns"] return (2, form_xmlns, app_id) forms = sorted(forms, key=_sortkey) # if there is a custom group export defined grab it here groups = HQGroupExportConfiguration.by_domain(self.domain) context = super(ExcelExportReport, self).report_context # Check if any custom exports are in the size hash saved_exports_has_media = any((e.app_id, e.index[1]) in size_hash for e in context["saved_exports"]) context.update( forms=forms, edit=self.request.GET.get("edit") == "true", group_exports=[group.form_exports for group in groups if group.form_exports], group_export_cutoff=datetime.utcnow() - timedelta(days=settings.SAVED_EXPORT_ACCESS_CUTOFF), report_slug=self.slug, property_hash=self.properties(size_hash), exports_has_media=size_hash, saved_exports_has_media=saved_exports_has_media, ) return context
def export_for_group_async(group_config_id): # exclude exports not accessed within the last 7 days last_access_cutoff = datetime.utcnow() - timedelta( days=settings.SAVED_EXPORT_ACCESS_CUTOFF) group_config = HQGroupExportConfiguration.get(group_config_id) export_for_group(group_config, last_access_cutoff=last_access_cutoff)
def _get_daily_saved_export_ids(domain): group_config = HQGroupExportConfiguration.get_for_domain(domain) return set(group_config.custom_export_ids)
def saved_exports(): for group_config in HQGroupExportConfiguration.view( "groupexport/by_domain", reduce=False, include_docs=True).all(): export_for_group(group_config, "couch")