def active_users(request): keys = [] number_threshold = 15 date_threshold_days_ago = 90 date_threshold = json_format_datetime(datetime.utcnow() - timedelta(days=date_threshold_days_ago)) key = make_form_couch_key(None, user_id="") for line in get_db().view("reports_forms/all_forms", startkey=key, endkey=key + [{}], group_level=3): if line["value"] >= number_threshold: keys.append(line["key"]) final_count = defaultdict(int) def is_valid_user_id(user_id): if not user_id: return False try: get_db().get(user_id) return True except Exception: return False for time_type, domain, user_id in keys: if get_db().view( "reports_forms/all_forms", reduce=False, startkey=[time_type, domain, user_id, date_threshold], limit=1 ): if True or is_valid_user_id(user_id): final_count[domain] += 1 return json_response({"break_down": final_count, "total": sum(final_count.values())})
def all_relevant_forms(self): selected_forms = FormsByApplicationFilter.get_value( self.request, self.domain) if self.request.GET.get('%s_unknown' % FormsByApplicationFilter.slug) == 'yes': return selected_forms # filter this result by submissions within this time frame key = make_form_couch_key(self.domain, by_submission_time=getattr( self, 'by_submission_time', True)) data = get_db().view( 'reports_forms/all_forms', reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ).all() all_submitted_forms = set([ FormsByApplicationFilter.make_xmlns_app_key( d['value']['xmlns'], d['value']['app_id']) for d in data ]) relevant_forms = all_submitted_forms.intersection( set(selected_forms.keys())) all_submitted_xmlns = [d['value']['xmlns'] for d in data] fuzzy_xmlns = set([ k for k in selected_forms.keys() if (FormsByApplicationFilter.fuzzy_slug in k and FormsByApplicationFilter.split_xmlns_app_key( k, only_xmlns=True) in all_submitted_xmlns) ]) relevant_forms = relevant_forms.union(fuzzy_xmlns) return dict([(k, selected_forms[k]) for k in relevant_forms])
def handle(self, *args, **options): if len(args) < 1: raise CommandError('Usage: manage.py submit_forms <domain>') domain = args[0] key = make_form_couch_key(domain) submissions = XFormInstance.view('reports_forms/all_forms', startkey=key, endkey=key + [{}], include_docs=True) ids_by_username = dict() def get_id(username): if username in ids_by_username: return ids_by_username[username] else: userID = get_db().view('users/logins_by_username', key=username).one() userID = userID['value'] if userID else None ids_by_username[username] = userID return userID for submission in submissions: if 'meta' in submission.form: username = format_username(submission.form['meta']['username'], domain) userID = get_id(username) if userID: submission.form['meta']['userID'] = userID submission.save() print submission._id, username, userID else: print "skipped: %s" % submission._id
def handle(self, *args, **options): key = make_form_couch_key("hsph") test_forms = XFormInstance.view('reports_forms/all_forms', reduce=False, startkey=key, endkey=key+["2012-11-07"], include_docs=True, ).all() test_cases = CommCareCase.view('reports/case_activity', reduce=False, startkey=["","hsph"], endkey=["","hsph","2012-11-07"], include_docs=True, ).all() print "\nDELETING TEST HSPH FORMS" for form in test_forms: if form.domain == "hsph": sys.stdout.write(".") sys.stdout.flush() form.delete() print "\n\nDELETING TEST HSPH CASES" for case in test_cases: if case.domain == "hsph": sys.stdout.write(".") sys.stdout.flush() case.delete() print "\n"
def _all_domain_stats(): webuser_counts = defaultdict(lambda: 0) commcare_counts = defaultdict(lambda: 0) form_counts = defaultdict(lambda: 0) case_counts = defaultdict(lambda: 0) for row in get_db().view('users/by_domain', startkey=["active"], endkey=["active", {}], group_level=3).all(): _, domain, doc_type = row['key'] value = row['value'] { 'WebUser': webuser_counts, 'CommCareUser': commcare_counts }[doc_type][domain] = value key = make_form_couch_key(None) form_counts.update(dict([(row["key"][1], row["value"]) for row in \ get_db().view("reports_forms/all_forms", group=True, group_level=2, startkey=key, endkey=key+[{}] ).all()])) case_counts.update(dict([(row["key"][0], row["value"]) for row in \ get_db().view("hqcase/types_by_domain", group=True,group_level=1).all()])) return {"web_users": webuser_counts, "commcare_users": commcare_counts, "forms": form_counts, "cases": case_counts}
def all_relevant_forms(self): selected_forms = FormsByApplicationFilter.get_value(self.request, self.domain) if self.request.GET.get('%s_unknown' % FormsByApplicationFilter.slug) == 'yes': return selected_forms # filter this result by submissions within this time frame key = make_form_couch_key(self.domain, by_submission_time=getattr(self, 'by_submission_time', True)) data = get_db().view('reports_forms/all_forms', reduce=False, startkey=key+[self.datespan.startdate_param_utc], endkey=key+[self.datespan.enddate_param_utc], ).all() all_submitted_forms = set([FormsByApplicationFilter.make_xmlns_app_key(d['value']['xmlns'], d['value']['app_id']) for d in data]) relevant_forms = all_submitted_forms.intersection(set(selected_forms.keys())) all_submitted_xmlns = [d['value']['xmlns'] for d in data] fuzzy_xmlns = set([k for k in selected_forms.keys() if (FormsByApplicationFilter.fuzzy_slug in k and FormsByApplicationFilter.split_xmlns_app_key(k, only_xmlns=True) in all_submitted_xmlns)]) relevant_forms = relevant_forms.union(fuzzy_xmlns) return dict([(k, selected_forms[k]) for k in relevant_forms])
def delete_all_data(request, domain, template="cleanup/delete_all_data.html"): if request.method == 'GET': return render(request, template, { 'domain': domain }) key = make_form_couch_key(domain) xforms = XFormInstance.view('reports_forms/all_forms', startkey=key, endkey=key+[{}], include_docs=True, reduce=False ) cases = CommCareCase.view('case/by_date_modified', startkey=[domain, {}, {}], endkey=[domain, {}, {}, {}], include_docs=True, reduce=False ) suffix = DELETED_SUFFIX deletion_id = random_hex() for thing_list in (xforms, cases): for thing in thing_list: thing.doc_type += suffix thing['-deletion_id'] = deletion_id thing.save() return HttpResponseRedirect(reverse('homepage'))
def get_user_data(self, user_id): key = make_form_couch_key( self.domain, by_submission_time=False, user_id=user_id, xmlns=self.selected_xmlns["xmlns"], app_id=self.selected_xmlns["app_id"], ) data = ( get_db() .view( "reports_forms/all_forms", startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], reduce=False, ) .all() ) durations = [d["value"]["duration"] for d in data if d["value"]["duration"] is not None] error_msg = _("Problem retrieving form durations.") if (not durations and data) else None return { "count": len(data), "max": numpy.max(durations) if durations else None, "min": numpy.min(durations) if durations else None, "avg": numpy.average(durations) if durations else None, "med": numpy.median(durations) if durations else None, "std": numpy.std(durations) if durations else None, "durations": durations, "error_msg": error_msg, }
def activity_times(self): all_times = [] for user in self.users: for form, info in self.all_relevant_forms.items(): key = make_form_couch_key( self.domain, user_id=user.get("user_id"), xmlns=info["xmlns"], app_id=info["app_id"], by_submission_time=self.by_submission_time, ) data = ( get_db() .view( "reports_forms/all_forms", reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ) .all() ) all_times.extend([dateutil.parser.parse(d["key"][-1]) for d in data]) if self.by_submission_time: # completion time is assumed to be in the phone's timezone until we can send proper timezone info all_times = [tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone, self.timezone.zone) for t in all_times] return [(t.weekday(), t.hour) for t in all_times]
def get_value(self, user_ids, datespan=None): if datespan: enddate = datespan.enddate_utc else: enddate = datetime.datetime.utcnow() days = [] for user_id in user_ids: key = make_form_couch_key(self.domain, user_id=user_id) results = get_db().view("reports_forms/all_forms", reduce=False, include_docs=False, descending=True, startkey=key + [enddate.isoformat(), {}], endkey=key, limit=1).first() try: last_transmission = results['key'][-1] last_date = dateutil.parser.parse(last_transmission) last_date = last_date.replace(tzinfo=pytz.utc) enddate = enddate.replace(tzinfo=pytz.utc) td = enddate - last_date days.append(td.days) except Exception: pass if len(days) == 1: return days[0] if not days: return None return days
def get_form_data(self, user_id, xmlns, app_id): key = make_form_couch_key(self.domain, user_id=user_id, xmlns=xmlns, app_id=app_id) return get_db().view("reports_forms/all_forms", reduce=False, startkey=key+[self.datespan.startdate_param_utc], endkey=key+[self.datespan.enddate_param_utc] ).all()
def active_users(request): keys = [] number_threshold = 15 date_threshold_days_ago = 90 date_threshold = json_format_datetime(datetime.utcnow() - timedelta(days=date_threshold_days_ago)) key = make_form_couch_key(None, user_id="") for line in get_db().view("reports_forms/all_forms", startkey=key, endkey=key+[{}], group_level=3): if line['value'] >= number_threshold: keys.append(line["key"]) final_count = defaultdict(int) def is_valid_user_id(user_id): if not user_id: return False try: get_db().get(user_id) return True except Exception: return False for time_type, domain, user_id in keys: if get_db().view("reports_forms/all_forms", reduce=False, startkey=[time_type, domain, user_id, date_threshold], limit=1): if True or is_valid_user_id(user_id): final_count[domain] += 1 return json_response({"break_down": final_count, "total": sum(final_count.values())})
def get_user_data(self, user_id): key = make_form_couch_key(self.domain, by_submission_time=False, user_id=user_id, xmlns=self.selected_xmlns['xmlns'], app_id=self.selected_xmlns['app_id']) data = get_db().view("reports_forms/all_forms", startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], reduce=False).all() durations = [ d['value']['duration'] for d in data if d['value']['duration'] is not None ] error_msg = _("Problem retrieving form durations.") if ( not durations and data) else None return { "count": len(data), "max": numpy.max(durations) if durations else None, "min": numpy.min(durations) if durations else None, "avg": numpy.average(durations) if durations else None, "med": numpy.median(durations) if durations else None, "std": numpy.std(durations) if durations else None, "durations": durations, "error_msg": error_msg, }
def last_form_submission(domain, display=True): key = make_form_couch_key(domain) row = get_db().view("reports_forms/all_forms", reduce=False, startkey=key, endkey=key + [{}]).all() return display_time(row[-1], display) if row else "No forms"
def handle(self, *args, **options): key = make_form_couch_key("hsph") test_forms = XFormInstance.view( 'reports_forms/all_forms', reduce=False, startkey=key, endkey=key + ["2012-11-07"], include_docs=True, ).all() test_cases = CommCareCase.view( 'reports/case_activity', reduce=False, startkey=["", "hsph"], endkey=["", "hsph", "2012-11-07"], include_docs=True, ).all() print "\nDELETING TEST HSPH FORMS" for form in test_forms: if form.domain == "hsph": sys.stdout.write(".") sys.stdout.flush() form.delete() print "\n\nDELETING TEST HSPH CASES" for case in test_cases: if case.domain == "hsph": sys.stdout.write(".") sys.stdout.flush() case.delete()
def _cacheable_domain_activity_report(request): landmarks = json.loads(request.GET.get('landmarks') or "[7, 30, 90]") landmarks.sort() now = datetime.utcnow() dates = [] for landmark in landmarks: dates.append(now - timedelta(days=landmark)) domains = [{'name': domain.name, 'display_name': domain.display_name()} for domain in Domain.get_all()] for domain in domains: domain['users'] = dict([(user.user_id, {'raw_username': user.raw_username}) for user in CommCareUser.by_domain(domain['name'])]) if not domain['users']: continue key = make_form_couch_key(domain['name']) forms = [r['value'] for r in get_db().view('reports_forms/all_forms', reduce=False, startkey=key+[json_format_datetime(dates[-1])], endkey=key+[json_format_datetime(now)], ).all()] domain['user_sets'] = [dict() for landmark in landmarks] for form in forms: user_id = form.get('user_id') try: time = string_to_datetime(form['submission_time']).replace(tzinfo = None) except ValueError: continue if user_id in domain['users']: for i, date in enumerate(dates): if time > date: domain['user_sets'][i][user_id] = domain['users'][user_id] return HttpResponse(json.dumps({'domains': domains, 'landmarks': landmarks}))
def process_user(self, user): username = user.raw_username # initialize if username not in self.individual: self.emails.append(user.email) self.individual[username] = { 'strategy': self.schedule, 'game': self.schedule, } # process this week's forms key = make_form_couch_key(user.domain, user_id=user.user_id, xmlns=DAILY_DATA_XMLNS) forms = XFormInstance.view( "all_forms/view", startkey=key + [str(self.week[0])], endkey=key + [str(self.week[-1] + datetime.timedelta(days=1))], reduce=False, include_docs=True ).all() for form in forms: self.process_form(form, username) # get and extend weekly totals try: weekly_totals = self.last_week.individual[username]['weekly_totals'] except (LookupError, AttributeError): weekly_totals = [] weekly_totals.append([ self.week[0].strftime(DATE_FORMAT), sum([d for d in self.individual[username]['strategy'] if d>0]) ]) self.individual[username]['weekly_totals'] = weekly_totals
def get_value(self, user_ids, datespan=None, is_debug=False): if datespan: enddate = datespan.enddate_utc else: enddate = datetime.datetime.utcnow() days = [] for user_id in user_ids: key = make_form_couch_key(self.domain, user_id=user_id) results = XFormInstance.get_db().view("reports_forms/all_forms", reduce=False, include_docs=False, descending=True, startkey=key+[enddate.isoformat(), {}], endkey=key, limit=1 ).first() try: last_transmission = results['key'][-1] last_date = dateutil.parser.parse(last_transmission) last_date = last_date.replace(tzinfo=pytz.utc) enddate = enddate.replace(tzinfo=pytz.utc) td = enddate - last_date days.append(td.days) except Exception: pass if len(days) == 1: return days[0] if not days: return None return days
def activity_times(self): all_times = [] for user in self.users: for form, info in self.all_relevant_forms.items(): key = make_form_couch_key( self.domain, user_id=user.get('user_id'), xmlns=info['xmlns'], app_id=info['app_id'], by_submission_time=self.by_submission_time) data = get_db().view( "reports_forms/all_forms", reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ).all() all_times.extend( [dateutil.parser.parse(d['key'][-1]) for d in data]) if self.by_submission_time: # completion time is assumed to be in the phone's timezone until we can send proper timezone info all_times = [ tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone, self.timezone.zone) for t in all_times ] return [(t.weekday(), t.hour) for t in all_times]
def _all_domain_stats(): webuser_counts = defaultdict(lambda: 0) commcare_counts = defaultdict(lambda: 0) form_counts = defaultdict(lambda: 0) case_counts = defaultdict(lambda: 0) for row in get_db().view('users/by_domain', startkey=["active"], endkey=["active", {}], group_level=3).all(): _, domain, doc_type = row['key'] value = row['value'] { 'WebUser': webuser_counts, 'CommCareUser': commcare_counts }[doc_type][domain] = value key = make_form_couch_key(None) form_counts.update(dict([(row["key"][1], row["value"]) for row in \ get_db().view("reports_forms/all_forms", group=True, group_level=2, startkey=key, endkey=key+[{}] ).all()])) case_counts.update(get_number_of_cases_per_domain()) return { "web_users": webuser_counts, "commcare_users": commcare_counts, "forms": form_counts, "cases": case_counts }
def process_user(self, user): username = user.raw_username # initialize if username not in self.individual: self.emails.append(user.email) self.individual[username] = { 'strategy': self.schedule, 'game': self.schedule, } # process this week's forms key = make_form_couch_key(user.domain, user_id=user.user_id, xmlns=DAILY_DATA_XMLNS) forms = XFormInstance.view( "reports_forms/all_forms", startkey=key + [str(self.week[0])], endkey=key + [str(self.week[-1] + datetime.timedelta(days=1))], reduce=False, include_docs=True ).all() for form in forms: self.process_form(form, username) # get and extend weekly totals try: weekly_totals = self.last_week.individual[username]['weekly_totals'] except (LookupError, AttributeError): weekly_totals = [] weekly_totals.append([ self.week[0].strftime(DATE_FORMAT), sum([d for d in self.individual[username]['strategy'] if d>0]) ]) self.individual[username]['weekly_totals'] = weekly_totals
def handle(self, *args, **options): if len(args) < 1: raise CommandError('Usage: manage.py submit_forms <domain>') domain = args[0] key = make_form_couch_key(domain) submissions = XFormInstance.view('reports_forms/all_forms', startkey=key, endkey=key+[{}], include_docs=True ) ids_by_username = dict() def get_id(username): if username in ids_by_username: return ids_by_username[username] else: userID = get_db().view('users/logins_by_username', key=username).one() userID = userID['value'] if userID else None ids_by_username[username] = userID return userID for submission in submissions: if 'meta' in submission.form: username = format_username(submission.form['meta']['username'], domain) userID = get_id(username) if userID: submission.form['meta']['userID'] = userID submission.save() print submission._id, username, userID else: print "skipped: %s" % submission._id
def get_number_of_forms_in_domain(domain): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain) row = ( XFormInstance.get_db().view("reports_forms/all_forms", startkey=key, endkey=key + [{}], stale=stale_ok()).one() ) return row["value"] if row else 0
def last_form_submission(domain, display=True): key = make_form_couch_key(domain) row = ( get_db() .view("reports_forms/all_forms", reduce=False, endkey=key, startkey=key + [{}], descending=True, limit=1) .first() ) return display_time(row, display) if row else "No forms"
def _get_num_submissions(self, user_id, xmlns, app_id): key = make_form_couch_key(self.domain, user_id=user_id, xmlns=xmlns, app_id=app_id) data = get_db().view('reports_forms/all_forms', reduce=True, startkey=key+[self.datespan.startdate_param_utc], endkey=key+[self.datespan.enddate_param_utc], ).first() return data['value'] if data else 0
def get_all_vhnd_forms(self): key = make_form_couch_key(DOMAIN, xmlns=VHND_XMLNS) return get_db().view( 'reports_forms/all_forms', startkey=key, endkey=key+[{}], reduce=False, include_docs=True ).all()
def last_form_submission(domain, display=True): key = make_form_couch_key(domain) row = get_db().view("reports_forms/all_forms", reduce=False, endkey=key, startkey=key + [{}], descending=True, limit=1).first() return display_time(row, display) if row else "No forms"
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationFilter.slug, '') for user in self.users: last_seen = last_sync = app_name = None key = make_form_couch_key(self.domain, user_id=user.user_id, app_id=selected_app or None) xform = XFormInstance.view( "reports_forms/all_forms", startkey=key + [{}], endkey=key, include_docs=True, descending=True, reduce=False, limit=1, ).first() if xform: last_seen = xform.received_on build_version, build_version_source = get_build_version(xform) if xform.app_id: try: app = get_app(self.domain, xform.app_id) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform) build_html = _build_html(build_version, build_version_source) app_name = app_name or _("Unknown App") app_name = format_html( u'{} {}', app_name, mark_safe(build_html), ) if app_name is None and selected_app: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date rows.append([ user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---" ]) return rows
def get_form_data(self, user_id, xmlns, app_id): key = make_form_couch_key(self.domain, user_id=user_id, xmlns=xmlns, app_id=app_id) return get_db().view( "reports_forms/all_forms", reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc]).all()
def get_number_of_forms_per_domain(): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(None) return { row["key"][1]: row["value"] for row in XFormInstance.get_db() .view("reports_forms/all_forms", group=True, group_level=2, startkey=key, endkey=key + [{}], stale=stale_ok()) .all() }
def form_count(user_id): key = make_form_couch_key(self.domain, user_id=user_id) result = XFormInstance.view('reports_forms/all_forms', startkey=key, endkey=key + [{}], group_level=0).one() if result: return result['value'] else: return 0
def get_number_of_forms_in_domain(domain): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain) row = XFormInstance.get_db().view( "all_forms/view", startkey=key, endkey=key + [{}], stale=stale_ok(), ).one() return row["value"] if row else 0
def submissions_errors(request, template="hqadmin/submissions_errors_report.html"): show_dates = "true" datespan = request.datespan domains = Domain.get_all() rows = [] for domain in domains: key = ["active", domain.name] data = get_db().view('users/by_domain', startkey=key, endkey=key + [{}], reduce=True).all() num_active_users = data[0].get('value', 0) if data else 0 key = make_form_couch_key(domain.name) data = get_db().view('reports_forms/all_forms', startkey=key + [datespan.startdate_param_utc], endkey=key + [datespan.enddate_param_utc, {}], reduce=True).all() num_forms_submitted = data[0].get('value', 0) if data else 0 phonelogs = DeviceReportEntry.objects.filter( domain__exact=domain.name, date__range=[ datespan.startdate_param_utc, datespan.enddate_param_utc ]) num_errors = phonelogs.filter(type__in=TAGS["error"]).count() num_warnings = phonelogs.filter(type__in=TAGS["warning"]).count() rows.append( dict(domain=domain.name, active_users=num_active_users, submissions=num_forms_submitted, errors=num_errors, warnings=num_warnings)) context = get_hqadmin_base_context(request) context.update({ "show_dates": show_dates, "datespan": datespan, "layout_flush_content": True, "rows": rows }) headers = DataTablesHeader( DataTablesColumn("Domain"), DataTablesColumn("Active Users", sort_type=DTSortType.NUMERIC), DataTablesColumn("Forms Submitted", sort_type=DTSortType.NUMERIC), DataTablesColumn("Errors", sort_type=DTSortType.NUMERIC), DataTablesColumn("Warnings", sort_type=DTSortType.NUMERIC)) context["headers"] = headers context["aoColumns"] = headers.render_aoColumns return render(request, template, context)
def _household_verification_json( domain="dodoma", last_hvid_path=["household_verification"], next_hvid_path=["followup_id"], xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79', range=None): if range: start, end = map(string_to_datetime, range) else: now = datetime.utcnow() start, end = now - timedelta(days=7), now key = make_form_couch_key(domain, xmlns=xmlns) submissions = XFormInstance.view( 'reports_forms/all_forms', reduce=False, startkey=key + [json_format_datetime(start)], endkey=key + [json_format_datetime(end)], include_docs=True, ) stats = get_household_verification_data( submissions=submissions, next_hvid_path=next_hvid_path, last_hvid_path=last_hvid_path, ) stats_by_userID = {} for s in stats: stats_by_userID[s['userID']] = s s['username'] = "******" % s['userID'] users = CommCareUser.by_domain(domain) for user in users: userID = user.user_id username = user_id_to_username(userID) if userID in stats_by_userID: stats_by_userID[userID]['username'] = username else: stats.append({ 'userID': userID, 'username': username, 'total': 0, 'correct': 0 }) stats.sort(key=lambda s: s['username']) return { "headers": ["Username", "Correct", "Total", "Percent Correct"], "rows": [[ s['username'], s['correct'], s['total'], ("%s%%" % int(s['correct'] * 100 / s['total']) if s['total'] else "---") ] for s in stats], }
def get_num_forms_since(self, time): if not hasattr(self, 'domain'): self.domain = self.couch_user.domain if self.couch_user else None from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.domain, user_id=self.userID) r = get_db().view('reports_forms/all_forms', startkey=key + [json_format_datetime(time)], endkey=key + [{}], group=False).one() return r['value'] if r else 0
def form_count(user_id): key = make_form_couch_key(self.domain, user_id=user_id) result = XFormInstance.view('reports_forms/all_forms', startkey=key, endkey=key + [{}], group_level=0 ).one() if result: return result['value'] else: return 0
def _get_all_form_submissions(domain, startdate=None, enddate=None): key = make_form_couch_key(domain) startkey = key+[startdate] if startdate and enddate else key endkey = key+[enddate] if startdate and enddate else key + [{}] submissions = XFormInstance.view('reports_forms/all_forms', startkey=startkey, endkey=endkey, include_docs=True, reduce=False ) return submissions
def rows(self): key = make_form_couch_key(self.domain, by_submission_time=self.by_submission_time) results = get_db().view( "reports_forms/all_forms", reduce=False, startkey=key + [ self.datespan.startdate_param_utc if self.by_submission_time else self.datespan.startdate_param ], endkey=key + [ self.datespan.enddate_param_utc if self.by_submission_time else self.datespan.enddate_param ]).all() user_map = dict([(user.get('user_id'), i) for (i, user) in enumerate(self.users)]) date_map = dict([(date.strftime(DATE_FORMAT), i + 1) for (i, date) in enumerate(self.dates)]) rows = [[0] * (2 + len(date_map)) for _tmp in range(len(self.users))] total_row = [0] * (2 + len(date_map)) for result in results: _tmp, _domain, date = result['key'] date = dateutil.parser.parse(date) tz_offset = self.timezone.localize( self.datespan.enddate).strftime("%z") date = date + datetime.timedelta( hours=int(tz_offset[0:3]), minutes=int(tz_offset[0] + tz_offset[3:5])) date = date.isoformat() val = result['value'] user_id = val.get("user_id") if user_id in self.user_ids: date_key = date_map.get(date[0:10], None) if date_key: rows[user_map[user_id]][date_key] += 1 for i, user in enumerate(self.users): rows[i][0] = self.get_user_link(user) total = sum(rows[i][1:-1]) rows[i][-1] = total total_row[1:-1] = [ total_row[ind + 1] + val for ind, val in enumerate(rows[i][1:-1]) ] total_row[-1] += total total_row[0] = _("All Users") self.total_row = total_row for row in rows: row[1:] = [self.table_cell(val) for val in row[1:]] return rows
def submissions_errors(request, template="hqadmin/submissions_errors_report.html"): show_dates = "true" datespan = request.datespan domains = Domain.get_all() rows = [] for domain in domains: key = ["active", domain.name] data = get_db().view('users/by_domain', startkey=key, endkey=key+[{}], reduce=True ).all() num_active_users = data[0].get('value', 0) if data else 0 key = make_form_couch_key(domain.name) data = get_db().view('reports_forms/all_forms', startkey=key+[datespan.startdate_param_utc], endkey=key+[datespan.enddate_param_utc, {}], reduce=True ).all() num_forms_submitted = data[0].get('value', 0) if data else 0 phonelogs = DeviceReportEntry.objects.filter(domain__exact=domain.name, date__range=[datespan.startdate_param_utc, datespan.enddate_param_utc]) num_errors = phonelogs.filter(type__in=TAGS["error"]).count() num_warnings = phonelogs.filter(type__in=TAGS["warning"]).count() rows.append(dict(domain=domain.name, active_users=num_active_users, submissions=num_forms_submitted, errors=num_errors, warnings=num_warnings)) context = get_hqadmin_base_context(request) context.update({ "show_dates": show_dates, "datespan": datespan, "layout_flush_content": True, "rows": rows }) headers = DataTablesHeader( DataTablesColumn("Domain"), DataTablesColumn("Active Users", sort_type=DTSortType.NUMERIC), DataTablesColumn("Forms Submitted", sort_type=DTSortType.NUMERIC), DataTablesColumn("Errors", sort_type=DTSortType.NUMERIC), DataTablesColumn("Warnings", sort_type=DTSortType.NUMERIC) ) context["headers"] = headers context["aoColumns"] = headers.render_aoColumns return render(request, template, context)
def _get_num_submissions(self, user_id, xmlns, app_id): key = make_form_couch_key(self.domain, user_id=user_id, xmlns=xmlns, app_id=app_id) data = get_db().view( 'reports_forms/all_forms', reduce=True, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ).first() return data['value'] if data else 0
def get_num_forms_since(self, time): if not hasattr(self, 'domain'): self.domain = self.couch_user.domain if self.couch_user else None from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.domain, user_id=self.userID) r = get_db().view('reports_forms/all_forms', startkey=key+[json_format_datetime(time)], endkey=key+[{}], group=False ).one() return r['value'] if r else 0
def _household_verification_json( domain="dodoma", last_hvid_path=["household_verification"], next_hvid_path=["followup_id"], xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79', range=None ): if range: start, end = map(string_to_datetime, range) else: now = datetime.utcnow() start, end = now - timedelta(days=7), now key = make_form_couch_key(domain, xmlns=xmlns) submissions = XFormInstance.view('reports_forms/all_forms', reduce=False, startkey=key+[json_format_datetime(start)], endkey=key+[json_format_datetime(end)], include_docs=True, ) stats = get_household_verification_data( submissions=submissions, next_hvid_path=next_hvid_path, last_hvid_path=last_hvid_path, ) stats_by_userID = {} for s in stats: stats_by_userID[s['userID']] = s s['username'] = "******" % s['userID'] users = CommCareUser.by_domain(domain) for user in users: userID = user.user_id username = user_id_to_username(userID) if userID in stats_by_userID: stats_by_userID[userID]['username'] = username else: stats.append({'userID': userID, 'username': username, 'total': 0, 'correct': 0}) stats.sort(key=lambda s: s['username']) return { "headers": ["Username", "Correct", "Total", "Percent Correct"], "rows": [[ s['username'], s['correct'], s['total'], ("%s%%" % int(s['correct']*100/s['total']) if s['total'] else "---") ] for s in stats], }
def get_last_form_submission_by_xmlns(domain, xmlns): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain, xmlns=xmlns) return XFormInstance.view( "all_forms/view", reduce=False, endkey=key, startkey=key + [{}], descending=True, limit=1, include_docs=True, stale=stale_ok(), ).first()
def get_number_of_submissions(domain, user_id, xmlns, app_id, start, end, by_submission_time=True): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain, user_id=user_id, xmlns=xmlns, by_submission_time=by_submission_time, app_id=app_id) data = XFormInstance.get_db().view( 'reports_forms/all_forms', reduce=True, startkey=key + [json_format_datetime(start)], endkey=key + [json_format_datetime(end)], stale=stale_ok(), ).first() return data['value'] if data else 0
def get_first_form_submission_received(domain): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain) row = ( XFormInstance.get_db() .view("reports_forms/all_forms", reduce=False, startkey=key, endkey=key + [{}], limit=1, stale=stale_ok()) .first() ) if row: submission_time = iso_string_to_datetime(row["key"][2]) else: submission_time = None return submission_time
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationFilter.slug, '') for user in self.users: last_seen = last_sync = app_name = None key = make_form_couch_key(self.domain, user_id=user.user_id, app_id=selected_app or None) xform = XFormInstance.view( "reports_forms/all_forms", startkey=key+[{}], endkey=key, include_docs=True, descending=True, reduce=False, limit=1, ).first() if xform: last_seen = xform.received_on build_version, build_version_source = get_build_version(xform) if xform.app_id: try: app = get_app(self.domain, xform.app_id) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform) build_html = _build_html(build_version, build_version_source) app_name = app_name or _("Unknown App") app_name = format_html( u'{} {}', app_name, mark_safe(build_html), ) if app_name is None and selected_app: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date rows.append( [user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"] ) return rows
def rows(self): # for user_type in [HQUserType.DEMO_USER, HQUserType.ADMIN]: # if self.user_filter[user_type].show\ # and not HQUserType.human_readable[user_type] in self.usernames: # temp_user = TempCommCareUser(self.domain, HQUserType.human_readable[user_type], "unknownID") # self._users.append(temp_user) rows = [] query_string = self.request.META['QUERY_STRING'] child_report_url = DeviceLogDetailsReport.get_url(domain=self.domain) for user in self.users: key = [self.domain, "errors_only", user.get('raw_username')] data = get_db().view("phonelog/devicelog_data", reduce=True, startkey=key+[self.datespan.startdate_param_utc], endkey=key+[self.datespan.enddate_param_utc] ).first() warning_count = 0 error_count = 0 if data: data = data.get('value', {}) error_count = data.get('errors', 0) warning_count = data.get('warnings', 0) formatted_warning_count = '<span class="label label-warning">%d</span>' % warning_count if warning_count > 0\ else '<span class="label">%d</span>' % warning_count formatted_error_count = '<span class="label label-important">%d</span>' % error_count if error_count > 0\ else '<span class="label">%d</span>' % error_count from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.domain, user_id=user.get('user_id')) data = get_db().view("reports_forms/all_forms", startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc, {}], reduce=True ).all() form_count = data[0]['value'] if data else 0 username_formatted = '<a href="%(url)s?%(query_string)s%(error_slug)s=True&%(username_slug)s=%(raw_username)s">%(username)s</a>' % { "url": child_report_url, "error_slug": DeviceLogTagField.errors_only_slug, "username_slug": DeviceLogUsersField.slug, "username": user.get('username_in_report'), "raw_username": user.get('raw_username'), "query_string": "%s&" % query_string if query_string else "" } rows.append([self.table_cell(user.get('raw_username'), username_formatted), self.table_cell(form_count), self.table_cell(warning_count, formatted_warning_count), self.table_cell(error_count, formatted_error_count)]) return rows
def last_submit_time(user_id): #need to call it directly due to reversed not liking the keys set the regular way key = make_form_couch_key(self.domain, user_id=user_id) v = XFormInstance.get_db().view('all_forms/view', endkey=key, startkey=key + [{}], reduce=False, include_docs=False, descending=True, limit=1 ) res = v.one() if res is None: return None else: return iso_string_to_datetime(res['key'][3]).strftime("%m/%d/%Y")
def last_submit_time(user_id): #need to call it directly due to reversed not liking the keys set the regular way key = make_form_couch_key(self.domain, user_id=user_id) v = XFormInstance.get_db().view('reports_forms/all_forms', endkey=key, startkey=key + [{}], reduce=False, include_docs=False, descending=True, limit=1 ) res = v.one() if res is None: return None else: return iso_string_to_datetime(res['key'][3]).strftime("%m/%d/%Y")
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationField.slug, '') UNKNOWN = _("unknown") for user in self.users: last_seen = self.table_cell(-1, _("Never")) app_name = None key = make_form_couch_key(self.domain, user_id=user.get('user_id'), app_id=selected_app if selected_app else None) data = XFormInstance.view( "reports_forms/all_forms", startkey=key+[{}], endkey=key, include_docs=True, descending=True, reduce=False, limit=1, ).first() if data: last_seen = util.format_relative_date(data.received_on) if data.version != '1': build_id = data.version else: build_id = UNKNOWN if data.app_id: try: app = Application.get(data.app_id) app_name = "%s [%s]" % (app.name, build_id) except ResourceNotFound: pass else: try: form_data = data.get_form app_name = form_data['meta']['appVersion']['#text'] except KeyError: pass app_name = app_name or _("Unknown App") if app_name is None and selected_app: continue rows.append([user.get('username_in_report'), last_seen, app_name or "---"]) return rows
def recent_submissions(self): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.name) res = get_db().view('reports_forms/all_forms', startkey=key+[{}], endkey=key, descending=True, reduce=False, include_docs=False, limit=1).all() if len(res) > 0: # if there have been any submissions in the past 30 days return (datetime.now() <= datetime.strptime(res[0]['value']['submission_time'], "%Y-%m-%dT%H:%M:%SZ") + timedelta(days=30)) else: return False
def get_first_form_submission_received(domain): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain) row = XFormInstance.get_db().view( "all_forms/view", reduce=False, startkey=key, endkey=key + [{}], limit=1, stale=stale_ok(), ).first() if row: submission_time = iso_string_to_datetime(row["key"][2]) else: submission_time = None return submission_time
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationField.slug, '') UNKNOWN = _("unknown") for user in self.users: last_seen = self.table_cell(-1, _("Never")) app_name = "---" is_unknown = True key = make_form_couch_key(self.domain, user_id=user.get('user_id')) data = XFormInstance.view("reports_forms/all_forms", startkey=key+[{}], endkey=key, include_docs=True, descending=True, reduce=False, limit=1, ).first() if data: last_seen = util.format_relative_date(data.received_on) if data.version != '1': build_id = data.version else: build_id = UNKNOWN form_data = data.get_form try: app_name = form_data['meta']['appVersion']['#text'] except KeyError: try: app = Application.get(data.app_id) is_unknown = False if selected_app and selected_app != data.app_id: continue app_name = "%s [%s]" % (app.name, build_id) except Exception: app_name = UNKNOWN if is_unknown and selected_app: continue row = [user.get('username_in_report'), last_seen, app_name] rows.append(row) return rows
def default_datespan(self): datespan = super(FormExportReportBase, self).default_datespan def extract_date(x): try: def clip_timezone(datestring): return datestring[:len('yyyy-mm-ddThh:mm:ss')] return string_to_datetime(clip_timezone(x['key'][2])) except Exception: logging.error("Tried to get a date from this, but it didn't work: %r" % x) return None key = make_form_couch_key(self.domain) startdate = get_db().view('reports_forms/all_forms', startkey=key, endkey=key+[{}], limit=1, descending=False, reduce=False, wrapper=extract_date ).one() if startdate: datespan.startdate = startdate return datespan
def query(limit=100, userID=None, group=False, username__exclude=["demo_user", "admin"], **kwargs): if group: if userID is None: key = [domain] else: key = [domain, userID] subs = [dict( userID = r['key'][1], username = r['key'][2], deviceID = r['key'][3], submissions = r['value'], ) for r in get_db().view('cleanup/submissions', startkey = key, endkey = key + [{}], group = True, )] total = len(subs) else: if userID is None: key = make_form_couch_key(domain) subs = XFormInstance.view('reports_forms/all_forms', startkey=key+[{}], endkey=key, reduce=False, include_docs=True, descending=True, limit=limit ) total = get_db().view('reports_forms/all_forms', startkey=key, endkey=key+[{}], group_level=1 ).one() total = total['value'] if total else 0 else: key = make_form_couch_key(domain, user_id=userID) subs = XFormInstance.view('reports_forms/all_forms', startkey=key+[{}], endkey=key, reduce=False, include_docs=True, descending=True, limit=limit ) total = get_db().view('reports_forms/all_forms', startkey=key, endkey=key+[{}], group_level=2 ).one() total = total['value'] if total else 0 _subs = [] for s in subs: try: _subs.append(dict( username = s['form']['meta']['username'], userID = s['form']['meta']['userID'], received_on = unicode(s['received_on']), deviceID = s['form']['meta']['deviceID'], )) except: continue subs = _subs if username__exclude: username__exclude = set(username__exclude) subs = filter(lambda sub: sub['username'] not in username__exclude, subs) return json_response({ "results": subs, "total": total, })