def active_users(request):
keys = []
number_threshold = 15
date_threshold_days_ago = 90
date_threshold = json_format_datetime(datetime.utcnow() - timedelta(days=date_threshold_days_ago))
key = make_form_couch_key(None, user_id="")
for line in get_db().view("reports_forms/all_forms", startkey=key, endkey=key + [{}], group_level=3):
if line["value"] >= number_threshold:
keys.append(line["key"])
final_count = defaultdict(int)
def is_valid_user_id(user_id):
if not user_id:
return False
try:
get_db().get(user_id)
return True
except Exception:
return False
for time_type, domain, user_id in keys:
if get_db().view(
"reports_forms/all_forms", reduce=False, startkey=[time_type, domain, user_id, date_threshold], limit=1
):
if True or is_valid_user_id(user_id):
final_count[domain] += 1
return json_response({"break_down": final_count, "total": sum(final_count.values())})
def all_relevant_forms(self):
selected_forms = FormsByApplicationFilter.get_value(
self.request, self.domain)
if self.request.GET.get('%s_unknown' %
FormsByApplicationFilter.slug) == 'yes':
return selected_forms
# filter this result by submissions within this time frame
key = make_form_couch_key(self.domain,
by_submission_time=getattr(
self, 'by_submission_time', True))
data = get_db().view(
'reports_forms/all_forms',
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
).all()
all_submitted_forms = set([
FormsByApplicationFilter.make_xmlns_app_key(
d['value']['xmlns'], d['value']['app_id']) for d in data
])
relevant_forms = all_submitted_forms.intersection(
set(selected_forms.keys()))
all_submitted_xmlns = [d['value']['xmlns'] for d in data]
fuzzy_xmlns = set([
k for k in selected_forms.keys()
if (FormsByApplicationFilter.fuzzy_slug in k
and FormsByApplicationFilter.split_xmlns_app_key(
k, only_xmlns=True) in all_submitted_xmlns)
])
relevant_forms = relevant_forms.union(fuzzy_xmlns)
return dict([(k, selected_forms[k]) for k in relevant_forms])
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage: manage.py submit_forms <domain>')
domain = args[0]
key = make_form_couch_key(domain)
submissions = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key + [{}],
include_docs=True)
ids_by_username = dict()
def get_id(username):
if username in ids_by_username:
return ids_by_username[username]
else:
userID = get_db().view('users/logins_by_username',
key=username).one()
userID = userID['value'] if userID else None
ids_by_username[username] = userID
return userID
for submission in submissions:
if 'meta' in submission.form:
username = format_username(submission.form['meta']['username'],
domain)
userID = get_id(username)
if userID:
submission.form['meta']['userID'] = userID
submission.save()
print submission._id, username, userID
else:
print "skipped: %s" % submission._id
def handle(self, *args, **options):
key = make_form_couch_key("hsph")
test_forms = XFormInstance.view('reports_forms/all_forms',
reduce=False,
startkey=key,
endkey=key+["2012-11-07"],
include_docs=True,
).all()
test_cases = CommCareCase.view('reports/case_activity',
reduce=False,
startkey=["","hsph"],
endkey=["","hsph","2012-11-07"],
include_docs=True,
).all()
print "\nDELETING TEST HSPH FORMS"
for form in test_forms:
if form.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
form.delete()
print "\n\nDELETING TEST HSPH CASES"
for case in test_cases:
if case.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
case.delete()
print "\n"
def _all_domain_stats():
webuser_counts = defaultdict(lambda: 0)
commcare_counts = defaultdict(lambda: 0)
form_counts = defaultdict(lambda: 0)
case_counts = defaultdict(lambda: 0)
for row in get_db().view('users/by_domain', startkey=["active"],
endkey=["active", {}], group_level=3).all():
_, domain, doc_type = row['key']
value = row['value']
{
'WebUser': webuser_counts,
'CommCareUser': commcare_counts
}[doc_type][domain] = value
key = make_form_couch_key(None)
form_counts.update(dict([(row["key"][1], row["value"]) for row in \
get_db().view("reports_forms/all_forms",
group=True,
group_level=2,
startkey=key,
endkey=key+[{}]
).all()]))
case_counts.update(dict([(row["key"][0], row["value"]) for row in \
get_db().view("hqcase/types_by_domain",
group=True,group_level=1).all()]))
return {"web_users": webuser_counts,
"commcare_users": commcare_counts,
"forms": form_counts,
"cases": case_counts}
def all_relevant_forms(self):
selected_forms = FormsByApplicationFilter.get_value(self.request, self.domain)
if self.request.GET.get('%s_unknown' % FormsByApplicationFilter.slug) == 'yes':
return selected_forms
# filter this result by submissions within this time frame
key = make_form_couch_key(self.domain, by_submission_time=getattr(self, 'by_submission_time', True))
data = get_db().view('reports_forms/all_forms',
reduce=False,
startkey=key+[self.datespan.startdate_param_utc],
endkey=key+[self.datespan.enddate_param_utc],
).all()
all_submitted_forms = set([FormsByApplicationFilter.make_xmlns_app_key(d['value']['xmlns'], d['value']['app_id'])
for d in data])
relevant_forms = all_submitted_forms.intersection(set(selected_forms.keys()))
all_submitted_xmlns = [d['value']['xmlns'] for d in data]
fuzzy_xmlns = set([k for k in selected_forms.keys()
if (FormsByApplicationFilter.fuzzy_slug in k and
FormsByApplicationFilter.split_xmlns_app_key(k, only_xmlns=True) in all_submitted_xmlns)])
relevant_forms = relevant_forms.union(fuzzy_xmlns)
return dict([(k, selected_forms[k]) for k in relevant_forms])
def delete_all_data(request, domain, template="cleanup/delete_all_data.html"):
if request.method == 'GET':
return render(request, template, {
'domain': domain
})
key = make_form_couch_key(domain)
xforms = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
include_docs=True,
reduce=False
)
cases = CommCareCase.view('case/by_date_modified',
startkey=[domain, {}, {}],
endkey=[domain, {}, {}, {}],
include_docs=True,
reduce=False
)
suffix = DELETED_SUFFIX
deletion_id = random_hex()
for thing_list in (xforms, cases):
for thing in thing_list:
thing.doc_type += suffix
thing['-deletion_id'] = deletion_id
thing.save()
return HttpResponseRedirect(reverse('homepage'))
def get_user_data(self, user_id):
key = make_form_couch_key(
self.domain,
by_submission_time=False,
user_id=user_id,
xmlns=self.selected_xmlns["xmlns"],
app_id=self.selected_xmlns["app_id"],
)
data = (
get_db()
.view(
"reports_forms/all_forms",
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
reduce=False,
)
.all()
)
durations = [d["value"]["duration"] for d in data if d["value"]["duration"] is not None]
error_msg = _("Problem retrieving form durations.") if (not durations and data) else None
return {
"count": len(data),
"max": numpy.max(durations) if durations else None,
"min": numpy.min(durations) if durations else None,
"avg": numpy.average(durations) if durations else None,
"med": numpy.median(durations) if durations else None,
"std": numpy.std(durations) if durations else None,
"durations": durations,
"error_msg": error_msg,
}
def activity_times(self):
all_times = []
for user in self.users:
for form, info in self.all_relevant_forms.items():
key = make_form_couch_key(
self.domain,
user_id=user.get("user_id"),
xmlns=info["xmlns"],
app_id=info["app_id"],
by_submission_time=self.by_submission_time,
)
data = (
get_db()
.view(
"reports_forms/all_forms",
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
)
.all()
)
all_times.extend([dateutil.parser.parse(d["key"][-1]) for d in data])
if self.by_submission_time:
# completion time is assumed to be in the phone's timezone until we can send proper timezone info
all_times = [tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone, self.timezone.zone) for t in all_times]
return [(t.weekday(), t.hour) for t in all_times]
def get_value(self, user_ids, datespan=None):
if datespan:
enddate = datespan.enddate_utc
else:
enddate = datetime.datetime.utcnow()
days = []
for user_id in user_ids:
key = make_form_couch_key(self.domain, user_id=user_id)
results = get_db().view("reports_forms/all_forms",
reduce=False,
include_docs=False,
descending=True,
startkey=key + [enddate.isoformat(), {}],
endkey=key,
limit=1).first()
try:
last_transmission = results['key'][-1]
last_date = dateutil.parser.parse(last_transmission)
last_date = last_date.replace(tzinfo=pytz.utc)
enddate = enddate.replace(tzinfo=pytz.utc)
td = enddate - last_date
days.append(td.days)
except Exception:
pass
if len(days) == 1:
return days[0]
if not days:
return None
return days
def get_form_data(self, user_id, xmlns, app_id):
key = make_form_couch_key(self.domain, user_id=user_id, xmlns=xmlns, app_id=app_id)
return get_db().view("reports_forms/all_forms",
reduce=False,
startkey=key+[self.datespan.startdate_param_utc],
endkey=key+[self.datespan.enddate_param_utc]
).all()
def active_users(request):
keys = []
number_threshold = 15
date_threshold_days_ago = 90
date_threshold = json_format_datetime(datetime.utcnow() - timedelta(days=date_threshold_days_ago))
key = make_form_couch_key(None, user_id="")
for line in get_db().view("reports_forms/all_forms",
startkey=key,
endkey=key+[{}],
group_level=3):
if line['value'] >= number_threshold:
keys.append(line["key"])
final_count = defaultdict(int)
def is_valid_user_id(user_id):
if not user_id: return False
try:
get_db().get(user_id)
return True
except Exception:
return False
for time_type, domain, user_id in keys:
if get_db().view("reports_forms/all_forms",
reduce=False,
startkey=[time_type, domain, user_id, date_threshold],
limit=1):
if True or is_valid_user_id(user_id):
final_count[domain] += 1
return json_response({"break_down": final_count, "total": sum(final_count.values())})
def get_user_data(self, user_id):
key = make_form_couch_key(self.domain,
by_submission_time=False,
user_id=user_id,
xmlns=self.selected_xmlns['xmlns'],
app_id=self.selected_xmlns['app_id'])
data = get_db().view("reports_forms/all_forms",
startkey=key +
[self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
reduce=False).all()
durations = [
d['value']['duration'] for d in data
if d['value']['duration'] is not None
]
error_msg = _("Problem retrieving form durations.") if (
not durations and data) else None
return {
"count": len(data),
"max": numpy.max(durations) if durations else None,
"min": numpy.min(durations) if durations else None,
"avg": numpy.average(durations) if durations else None,
"med": numpy.median(durations) if durations else None,
"std": numpy.std(durations) if durations else None,
"durations": durations,
"error_msg": error_msg,
}
def last_form_submission(domain, display=True):
key = make_form_couch_key(domain)
row = get_db().view("reports_forms/all_forms",
reduce=False,
startkey=key,
endkey=key + [{}]).all()
return display_time(row[-1], display) if row else "No forms"
def handle(self, *args, **options):
key = make_form_couch_key("hsph")
test_forms = XFormInstance.view(
'reports_forms/all_forms',
reduce=False,
startkey=key,
endkey=key + ["2012-11-07"],
include_docs=True,
).all()
test_cases = CommCareCase.view(
'reports/case_activity',
reduce=False,
startkey=["", "hsph"],
endkey=["", "hsph", "2012-11-07"],
include_docs=True,
).all()
print "\nDELETING TEST HSPH FORMS"
for form in test_forms:
if form.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
form.delete()
print "\n\nDELETING TEST HSPH CASES"
for case in test_cases:
if case.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
case.delete()
def _cacheable_domain_activity_report(request):
landmarks = json.loads(request.GET.get('landmarks') or "[7, 30, 90]")
landmarks.sort()
now = datetime.utcnow()
dates = []
for landmark in landmarks:
dates.append(now - timedelta(days=landmark))
domains = [{'name': domain.name, 'display_name': domain.display_name()} for domain in Domain.get_all()]
for domain in domains:
domain['users'] = dict([(user.user_id, {'raw_username': user.raw_username}) for user in CommCareUser.by_domain(domain['name'])])
if not domain['users']:
continue
key = make_form_couch_key(domain['name'])
forms = [r['value'] for r in get_db().view('reports_forms/all_forms',
reduce=False,
startkey=key+[json_format_datetime(dates[-1])],
endkey=key+[json_format_datetime(now)],
).all()]
domain['user_sets'] = [dict() for landmark in landmarks]
for form in forms:
user_id = form.get('user_id')
try:
time = string_to_datetime(form['submission_time']).replace(tzinfo = None)
except ValueError:
continue
if user_id in domain['users']:
for i, date in enumerate(dates):
if time > date:
domain['user_sets'][i][user_id] = domain['users'][user_id]
return HttpResponse(json.dumps({'domains': domains, 'landmarks': landmarks}))
def process_user(self, user):
username = user.raw_username
# initialize
if username not in self.individual:
self.emails.append(user.email)
self.individual[username] = {
'strategy': self.schedule,
'game': self.schedule,
}
# process this week's forms
key = make_form_couch_key(user.domain, user_id=user.user_id, xmlns=DAILY_DATA_XMLNS)
forms = XFormInstance.view(
"all_forms/view",
startkey=key + [str(self.week[0])],
endkey=key + [str(self.week[-1] + datetime.timedelta(days=1))],
reduce=False,
include_docs=True
).all()
for form in forms:
self.process_form(form, username)
# get and extend weekly totals
try:
weekly_totals = self.last_week.individual[username]['weekly_totals']
except (LookupError, AttributeError):
weekly_totals = []
weekly_totals.append([
self.week[0].strftime(DATE_FORMAT),
sum([d for d in self.individual[username]['strategy'] if d>0])
])
self.individual[username]['weekly_totals'] = weekly_totals
def get_value(self, user_ids, datespan=None, is_debug=False):
if datespan:
enddate = datespan.enddate_utc
else:
enddate = datetime.datetime.utcnow()
days = []
for user_id in user_ids:
key = make_form_couch_key(self.domain, user_id=user_id)
results = XFormInstance.get_db().view("reports_forms/all_forms",
reduce=False,
include_docs=False,
descending=True,
startkey=key+[enddate.isoformat(), {}],
endkey=key,
limit=1
).first()
try:
last_transmission = results['key'][-1]
last_date = dateutil.parser.parse(last_transmission)
last_date = last_date.replace(tzinfo=pytz.utc)
enddate = enddate.replace(tzinfo=pytz.utc)
td = enddate - last_date
days.append(td.days)
except Exception:
pass
if len(days) == 1:
return days[0]
if not days:
return None
return days
def activity_times(self):
all_times = []
for user in self.users:
for form, info in self.all_relevant_forms.items():
key = make_form_couch_key(
self.domain,
user_id=user.get('user_id'),
xmlns=info['xmlns'],
app_id=info['app_id'],
by_submission_time=self.by_submission_time)
data = get_db().view(
"reports_forms/all_forms",
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
).all()
all_times.extend(
[dateutil.parser.parse(d['key'][-1]) for d in data])
if self.by_submission_time:
# completion time is assumed to be in the phone's timezone until we can send proper timezone info
all_times = [
tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone,
self.timezone.zone)
for t in all_times
]
return [(t.weekday(), t.hour) for t in all_times]
def _all_domain_stats():
webuser_counts = defaultdict(lambda: 0)
commcare_counts = defaultdict(lambda: 0)
form_counts = defaultdict(lambda: 0)
case_counts = defaultdict(lambda: 0)
for row in get_db().view('users/by_domain',
startkey=["active"],
endkey=["active", {}],
group_level=3).all():
_, domain, doc_type = row['key']
value = row['value']
{
'WebUser': webuser_counts,
'CommCareUser': commcare_counts
}[doc_type][domain] = value
key = make_form_couch_key(None)
form_counts.update(dict([(row["key"][1], row["value"]) for row in \
get_db().view("reports_forms/all_forms",
group=True,
group_level=2,
startkey=key,
endkey=key+[{}]
).all()]))
case_counts.update(get_number_of_cases_per_domain())
return {
"web_users": webuser_counts,
"commcare_users": commcare_counts,
"forms": form_counts,
"cases": case_counts
}
def _cacheable_domain_activity_report(request):
landmarks = json.loads(request.GET.get('landmarks') or "[7, 30, 90]")
landmarks.sort()
now = datetime.utcnow()
dates = []
for landmark in landmarks:
dates.append(now - timedelta(days=landmark))
domains = [{'name': domain.name, 'display_name': domain.display_name()} for domain in Domain.get_all()]
for domain in domains:
domain['users'] = dict([(user.user_id, {'raw_username': user.raw_username}) for user in CommCareUser.by_domain(domain['name'])])
if not domain['users']:
continue
key = make_form_couch_key(domain['name'])
forms = [r['value'] for r in get_db().view('reports_forms/all_forms',
reduce=False,
startkey=key+[json_format_datetime(dates[-1])],
endkey=key+[json_format_datetime(now)],
).all()]
domain['user_sets'] = [dict() for landmark in landmarks]
for form in forms:
user_id = form.get('user_id')
try:
time = string_to_datetime(form['submission_time']).replace(tzinfo = None)
except ValueError:
continue
if user_id in domain['users']:
for i, date in enumerate(dates):
if time > date:
domain['user_sets'][i][user_id] = domain['users'][user_id]
return HttpResponse(json.dumps({'domains': domains, 'landmarks': landmarks}))
def process_user(self, user):
username = user.raw_username
# initialize
if username not in self.individual:
self.emails.append(user.email)
self.individual[username] = {
'strategy': self.schedule,
'game': self.schedule,
}
# process this week's forms
key = make_form_couch_key(user.domain, user_id=user.user_id, xmlns=DAILY_DATA_XMLNS)
forms = XFormInstance.view(
"reports_forms/all_forms",
startkey=key + [str(self.week[0])],
endkey=key + [str(self.week[-1] + datetime.timedelta(days=1))],
reduce=False,
include_docs=True
).all()
for form in forms:
self.process_form(form, username)
# get and extend weekly totals
try:
weekly_totals = self.last_week.individual[username]['weekly_totals']
except (LookupError, AttributeError):
weekly_totals = []
weekly_totals.append([
self.week[0].strftime(DATE_FORMAT),
sum([d for d in self.individual[username]['strategy'] if d>0])
])
self.individual[username]['weekly_totals'] = weekly_totals
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage: manage.py submit_forms <domain>')
domain = args[0]
key = make_form_couch_key(domain)
submissions = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
include_docs=True
)
ids_by_username = dict()
def get_id(username):
if username in ids_by_username:
return ids_by_username[username]
else:
userID = get_db().view('users/logins_by_username', key=username).one()
userID = userID['value'] if userID else None
ids_by_username[username] = userID
return userID
for submission in submissions:
if 'meta' in submission.form:
username = format_username(submission.form['meta']['username'], domain)
userID = get_id(username)
if userID:
submission.form['meta']['userID'] = userID
submission.save()
print submission._id, username, userID
else:
print "skipped: %s" % submission._id
def get_number_of_forms_in_domain(domain):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain)
row = (
XFormInstance.get_db().view("reports_forms/all_forms", startkey=key, endkey=key + [{}], stale=stale_ok()).one()
)
return row["value"] if row else 0
def last_form_submission(domain, display=True):
key = make_form_couch_key(domain)
row = (
get_db()
.view("reports_forms/all_forms", reduce=False, endkey=key, startkey=key + [{}], descending=True, limit=1)
.first()
)
return display_time(row, display) if row else "No forms"
def _get_num_submissions(self, user_id, xmlns, app_id):
key = make_form_couch_key(self.domain, user_id=user_id, xmlns=xmlns, app_id=app_id)
data = get_db().view('reports_forms/all_forms',
reduce=True,
startkey=key+[self.datespan.startdate_param_utc],
endkey=key+[self.datespan.enddate_param_utc],
).first()
return data['value'] if data else 0
def get_all_vhnd_forms(self):
key = make_form_couch_key(DOMAIN, xmlns=VHND_XMLNS)
return get_db().view(
'reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
reduce=False,
include_docs=True
).all()
def last_form_submission(domain, display=True):
key = make_form_couch_key(domain)
row = get_db().view("reports_forms/all_forms",
reduce=False,
endkey=key,
startkey=key + [{}],
descending=True,
limit=1).first()
return display_time(row, display) if row else "No forms"
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationFilter.slug,
'')
for user in self.users:
last_seen = last_sync = app_name = None
key = make_form_couch_key(self.domain,
user_id=user.user_id,
app_id=selected_app or None)
xform = XFormInstance.view(
"reports_forms/all_forms",
startkey=key + [{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if xform:
last_seen = xform.received_on
build_version, build_version_source = get_build_version(xform)
if xform.app_id:
try:
app = get_app(self.domain, xform.app_id)
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform)
build_html = _build_html(build_version, build_version_source)
app_name = app_name or _("Unknown App")
app_name = format_html(
u'{} {}',
app_name,
mark_safe(build_html),
)
if app_name is None and selected_app:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append([
user.username_in_report,
_fmt_date(last_seen),
_fmt_date(last_sync), app_name or "---"
])
return rows
def get_form_data(self, user_id, xmlns, app_id):
key = make_form_couch_key(self.domain,
user_id=user_id,
xmlns=xmlns,
app_id=app_id)
return get_db().view(
"reports_forms/all_forms",
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc]).all()
def get_number_of_forms_per_domain():
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(None)
return {
row["key"][1]: row["value"]
for row in XFormInstance.get_db()
.view("reports_forms/all_forms", group=True, group_level=2, startkey=key, endkey=key + [{}], stale=stale_ok())
.all()
}
def form_count(user_id):
key = make_form_couch_key(self.domain, user_id=user_id)
result = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key + [{}],
group_level=0).one()
if result:
return result['value']
else:
return 0
def get_number_of_forms_in_domain(domain):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain)
row = XFormInstance.get_db().view(
"all_forms/view",
startkey=key,
endkey=key + [{}],
stale=stale_ok(),
).one()
return row["value"] if row else 0
def submissions_errors(request,
template="hqadmin/submissions_errors_report.html"):
show_dates = "true"
datespan = request.datespan
domains = Domain.get_all()
rows = []
for domain in domains:
key = ["active", domain.name]
data = get_db().view('users/by_domain',
startkey=key,
endkey=key + [{}],
reduce=True).all()
num_active_users = data[0].get('value', 0) if data else 0
key = make_form_couch_key(domain.name)
data = get_db().view('reports_forms/all_forms',
startkey=key + [datespan.startdate_param_utc],
endkey=key + [datespan.enddate_param_utc, {}],
reduce=True).all()
num_forms_submitted = data[0].get('value', 0) if data else 0
phonelogs = DeviceReportEntry.objects.filter(
domain__exact=domain.name,
date__range=[
datespan.startdate_param_utc, datespan.enddate_param_utc
])
num_errors = phonelogs.filter(type__in=TAGS["error"]).count()
num_warnings = phonelogs.filter(type__in=TAGS["warning"]).count()
rows.append(
dict(domain=domain.name,
active_users=num_active_users,
submissions=num_forms_submitted,
errors=num_errors,
warnings=num_warnings))
context = get_hqadmin_base_context(request)
context.update({
"show_dates": show_dates,
"datespan": datespan,
"layout_flush_content": True,
"rows": rows
})
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("Active Users", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Forms Submitted", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Errors", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Warnings", sort_type=DTSortType.NUMERIC))
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
return render(request, template, context)
def _household_verification_json(
domain="dodoma",
last_hvid_path=["household_verification"],
next_hvid_path=["followup_id"],
xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79',
range=None):
if range:
start, end = map(string_to_datetime, range)
else:
now = datetime.utcnow()
start, end = now - timedelta(days=7), now
key = make_form_couch_key(domain, xmlns=xmlns)
submissions = XFormInstance.view(
'reports_forms/all_forms',
reduce=False,
startkey=key + [json_format_datetime(start)],
endkey=key + [json_format_datetime(end)],
include_docs=True,
)
stats = get_household_verification_data(
submissions=submissions,
next_hvid_path=next_hvid_path,
last_hvid_path=last_hvid_path,
)
stats_by_userID = {}
for s in stats:
stats_by_userID[s['userID']] = s
s['username'] = "******" % s['userID']
users = CommCareUser.by_domain(domain)
for user in users:
userID = user.user_id
username = user_id_to_username(userID)
if userID in stats_by_userID:
stats_by_userID[userID]['username'] = username
else:
stats.append({
'userID': userID,
'username': username,
'total': 0,
'correct': 0
})
stats.sort(key=lambda s: s['username'])
return {
"headers": ["Username", "Correct", "Total", "Percent Correct"],
"rows": [[
s['username'], s['correct'], s['total'],
("%s%%" %
int(s['correct'] * 100 / s['total']) if s['total'] else "---")
] for s in stats],
}
def get_num_forms_since(self, time):
if not hasattr(self, 'domain'):
self.domain = self.couch_user.domain if self.couch_user else None
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.domain, user_id=self.userID)
r = get_db().view('reports_forms/all_forms',
startkey=key + [json_format_datetime(time)],
endkey=key + [{}],
group=False).one()
return r['value'] if r else 0
def form_count(user_id):
key = make_form_couch_key(self.domain, user_id=user_id)
result = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key + [{}],
group_level=0
).one()
if result:
return result['value']
else:
return 0
def _get_all_form_submissions(domain, startdate=None, enddate=None):
key = make_form_couch_key(domain)
startkey = key+[startdate] if startdate and enddate else key
endkey = key+[enddate] if startdate and enddate else key + [{}]
submissions = XFormInstance.view('reports_forms/all_forms',
startkey=startkey,
endkey=endkey,
include_docs=True,
reduce=False
)
return submissions
def rows(self):
key = make_form_couch_key(self.domain,
by_submission_time=self.by_submission_time)
results = get_db().view(
"reports_forms/all_forms",
reduce=False,
startkey=key + [
self.datespan.startdate_param_utc
if self.by_submission_time else self.datespan.startdate_param
],
endkey=key + [
self.datespan.enddate_param_utc
if self.by_submission_time else self.datespan.enddate_param
]).all()
user_map = dict([(user.get('user_id'), i)
for (i, user) in enumerate(self.users)])
date_map = dict([(date.strftime(DATE_FORMAT), i + 1)
for (i, date) in enumerate(self.dates)])
rows = [[0] * (2 + len(date_map)) for _tmp in range(len(self.users))]
total_row = [0] * (2 + len(date_map))
for result in results:
_tmp, _domain, date = result['key']
date = dateutil.parser.parse(date)
tz_offset = self.timezone.localize(
self.datespan.enddate).strftime("%z")
date = date + datetime.timedelta(
hours=int(tz_offset[0:3]),
minutes=int(tz_offset[0] + tz_offset[3:5]))
date = date.isoformat()
val = result['value']
user_id = val.get("user_id")
if user_id in self.user_ids:
date_key = date_map.get(date[0:10], None)
if date_key:
rows[user_map[user_id]][date_key] += 1
for i, user in enumerate(self.users):
rows[i][0] = self.get_user_link(user)
total = sum(rows[i][1:-1])
rows[i][-1] = total
total_row[1:-1] = [
total_row[ind + 1] + val
for ind, val in enumerate(rows[i][1:-1])
]
total_row[-1] += total
total_row[0] = _("All Users")
self.total_row = total_row
for row in rows:
row[1:] = [self.table_cell(val) for val in row[1:]]
return rows
def submissions_errors(request, template="hqadmin/submissions_errors_report.html"):
show_dates = "true"
datespan = request.datespan
domains = Domain.get_all()
rows = []
for domain in domains:
key = ["active", domain.name]
data = get_db().view('users/by_domain',
startkey=key,
endkey=key+[{}],
reduce=True
).all()
num_active_users = data[0].get('value', 0) if data else 0
key = make_form_couch_key(domain.name)
data = get_db().view('reports_forms/all_forms',
startkey=key+[datespan.startdate_param_utc],
endkey=key+[datespan.enddate_param_utc, {}],
reduce=True
).all()
num_forms_submitted = data[0].get('value', 0) if data else 0
phonelogs = DeviceReportEntry.objects.filter(domain__exact=domain.name,
date__range=[datespan.startdate_param_utc, datespan.enddate_param_utc])
num_errors = phonelogs.filter(type__in=TAGS["error"]).count()
num_warnings = phonelogs.filter(type__in=TAGS["warning"]).count()
rows.append(dict(domain=domain.name,
active_users=num_active_users,
submissions=num_forms_submitted,
errors=num_errors,
warnings=num_warnings))
context = get_hqadmin_base_context(request)
context.update({
"show_dates": show_dates,
"datespan": datespan,
"layout_flush_content": True,
"rows": rows
})
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("Active Users", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Forms Submitted", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Errors", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Warnings", sort_type=DTSortType.NUMERIC)
)
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
return render(request, template, context)
def _get_num_submissions(self, user_id, xmlns, app_id):
key = make_form_couch_key(self.domain,
user_id=user_id,
xmlns=xmlns,
app_id=app_id)
data = get_db().view(
'reports_forms/all_forms',
reduce=True,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
).first()
return data['value'] if data else 0
def get_num_forms_since(self, time):
if not hasattr(self, 'domain'):
self.domain = self.couch_user.domain if self.couch_user else None
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.domain, user_id=self.userID)
r = get_db().view('reports_forms/all_forms',
startkey=key+[json_format_datetime(time)],
endkey=key+[{}],
group=False
).one()
return r['value'] if r else 0
def _household_verification_json(
domain="dodoma",
last_hvid_path=["household_verification"],
next_hvid_path=["followup_id"],
xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79',
range=None
):
if range:
start, end = map(string_to_datetime, range)
else:
now = datetime.utcnow()
start, end = now - timedelta(days=7), now
key = make_form_couch_key(domain, xmlns=xmlns)
submissions = XFormInstance.view('reports_forms/all_forms',
reduce=False,
startkey=key+[json_format_datetime(start)],
endkey=key+[json_format_datetime(end)],
include_docs=True,
)
stats = get_household_verification_data(
submissions=submissions,
next_hvid_path=next_hvid_path,
last_hvid_path=last_hvid_path,
)
stats_by_userID = {}
for s in stats:
stats_by_userID[s['userID']] = s
s['username'] = "******" % s['userID']
users = CommCareUser.by_domain(domain)
for user in users:
userID = user.user_id
username = user_id_to_username(userID)
if userID in stats_by_userID:
stats_by_userID[userID]['username'] = username
else:
stats.append({'userID': userID, 'username': username, 'total': 0, 'correct': 0})
stats.sort(key=lambda s: s['username'])
return {
"headers": ["Username", "Correct", "Total", "Percent Correct"],
"rows": [[
s['username'],
s['correct'],
s['total'],
("%s%%" % int(s['correct']*100/s['total']) if s['total'] else "---")
] for s in stats],
}
def get_last_form_submission_by_xmlns(domain, xmlns):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain, xmlns=xmlns)
return XFormInstance.view(
"all_forms/view",
reduce=False,
endkey=key,
startkey=key + [{}],
descending=True,
limit=1,
include_docs=True,
stale=stale_ok(),
).first()
def get_number_of_submissions(domain, user_id, xmlns, app_id, start, end,
by_submission_time=True):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain, user_id=user_id, xmlns=xmlns,
by_submission_time=by_submission_time,
app_id=app_id)
data = XFormInstance.get_db().view(
'reports_forms/all_forms',
reduce=True,
startkey=key + [json_format_datetime(start)],
endkey=key + [json_format_datetime(end)],
stale=stale_ok(),
).first()
return data['value'] if data else 0
def get_first_form_submission_received(domain):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain)
row = (
XFormInstance.get_db()
.view("reports_forms/all_forms", reduce=False, startkey=key, endkey=key + [{}], limit=1, stale=stale_ok())
.first()
)
if row:
submission_time = iso_string_to_datetime(row["key"][2])
else:
submission_time = None
return submission_time
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationFilter.slug, '')
for user in self.users:
last_seen = last_sync = app_name = None
key = make_form_couch_key(self.domain, user_id=user.user_id,
app_id=selected_app or None)
xform = XFormInstance.view(
"reports_forms/all_forms",
startkey=key+[{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if xform:
last_seen = xform.received_on
build_version, build_version_source = get_build_version(xform)
if xform.app_id:
try:
app = get_app(self.domain, xform.app_id)
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform)
build_html = _build_html(build_version, build_version_source)
app_name = app_name or _("Unknown App")
app_name = format_html(
u'{} {}', app_name, mark_safe(build_html),
)
if app_name is None and selected_app:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append(
[user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"]
)
return rows
def rows(self):
# for user_type in [HQUserType.DEMO_USER, HQUserType.ADMIN]:
# if self.user_filter[user_type].show\
# and not HQUserType.human_readable[user_type] in self.usernames:
# temp_user = TempCommCareUser(self.domain, HQUserType.human_readable[user_type], "unknownID")
# self._users.append(temp_user)
rows = []
query_string = self.request.META['QUERY_STRING']
child_report_url = DeviceLogDetailsReport.get_url(domain=self.domain)
for user in self.users:
key = [self.domain, "errors_only", user.get('raw_username')]
data = get_db().view("phonelog/devicelog_data",
reduce=True,
startkey=key+[self.datespan.startdate_param_utc],
endkey=key+[self.datespan.enddate_param_utc]
).first()
warning_count = 0
error_count = 0
if data:
data = data.get('value', {})
error_count = data.get('errors', 0)
warning_count = data.get('warnings', 0)
formatted_warning_count = '<span class="label label-warning">%d</span>' % warning_count if warning_count > 0\
else '<span class="label">%d</span>' % warning_count
formatted_error_count = '<span class="label label-important">%d</span>' % error_count if error_count > 0\
else '<span class="label">%d</span>' % error_count
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.domain, user_id=user.get('user_id'))
data = get_db().view("reports_forms/all_forms",
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc, {}],
reduce=True
).all()
form_count = data[0]['value'] if data else 0
username_formatted = '<a href="%(url)s?%(query_string)s%(error_slug)s=True&%(username_slug)s=%(raw_username)s">%(username)s</a>' % {
"url": child_report_url,
"error_slug": DeviceLogTagField.errors_only_slug,
"username_slug": DeviceLogUsersField.slug,
"username": user.get('username_in_report'),
"raw_username": user.get('raw_username'),
"query_string": "%s&" % query_string if query_string else ""
}
rows.append([self.table_cell(user.get('raw_username'), username_formatted),
self.table_cell(form_count),
self.table_cell(warning_count, formatted_warning_count),
self.table_cell(error_count, formatted_error_count)])
return rows
def last_submit_time(user_id):
#need to call it directly due to reversed not liking the keys set the regular way
key = make_form_couch_key(self.domain, user_id=user_id)
v = XFormInstance.get_db().view('all_forms/view',
endkey=key,
startkey=key + [{}],
reduce=False,
include_docs=False,
descending=True, limit=1
)
res = v.one()
if res is None:
return None
else:
return iso_string_to_datetime(res['key'][3]).strftime("%m/%d/%Y")
def last_submit_time(user_id):
#need to call it directly due to reversed not liking the keys set the regular way
key = make_form_couch_key(self.domain, user_id=user_id)
v = XFormInstance.get_db().view('reports_forms/all_forms',
endkey=key,
startkey=key + [{}],
reduce=False,
include_docs=False,
descending=True, limit=1
)
res = v.one()
if res is None:
return None
else:
return iso_string_to_datetime(res['key'][3]).strftime("%m/%d/%Y")
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationField.slug, '')
UNKNOWN = _("unknown")
for user in self.users:
last_seen = self.table_cell(-1, _("Never"))
app_name = None
key = make_form_couch_key(self.domain, user_id=user.get('user_id'), app_id=selected_app if selected_app else None)
data = XFormInstance.view(
"reports_forms/all_forms",
startkey=key+[{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if data:
last_seen = util.format_relative_date(data.received_on)
if data.version != '1':
build_id = data.version
else:
build_id = UNKNOWN
if data.app_id:
try:
app = Application.get(data.app_id)
app_name = "%s [%s]" % (app.name, build_id)
except ResourceNotFound:
pass
else:
try:
form_data = data.get_form
app_name = form_data['meta']['appVersion']['#text']
except KeyError:
pass
app_name = app_name or _("Unknown App")
if app_name is None and selected_app:
continue
rows.append([user.get('username_in_report'), last_seen, app_name or "---"])
return rows
def recent_submissions(self):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.name)
res = get_db().view('reports_forms/all_forms',
startkey=key+[{}],
endkey=key,
descending=True,
reduce=False,
include_docs=False,
limit=1).all()
if len(res) > 0: # if there have been any submissions in the past 30 days
return (datetime.now() <=
datetime.strptime(res[0]['value']['submission_time'], "%Y-%m-%dT%H:%M:%SZ")
+ timedelta(days=30))
else:
return False
def get_first_form_submission_received(domain):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain)
row = XFormInstance.get_db().view(
"all_forms/view",
reduce=False,
startkey=key,
endkey=key + [{}],
limit=1,
stale=stale_ok(),
).first()
if row:
submission_time = iso_string_to_datetime(row["key"][2])
else:
submission_time = None
return submission_time
def recent_submissions(self):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.name)
res = get_db().view('reports_forms/all_forms',
startkey=key+[{}],
endkey=key,
descending=True,
reduce=False,
include_docs=False,
limit=1).all()
if len(res) > 0: # if there have been any submissions in the past 30 days
return (datetime.now() <=
datetime.strptime(res[0]['value']['submission_time'], "%Y-%m-%dT%H:%M:%SZ")
+ timedelta(days=30))
else:
return False
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationField.slug, '')
UNKNOWN = _("unknown")
for user in self.users:
last_seen = self.table_cell(-1, _("Never"))
app_name = "---"
is_unknown = True
key = make_form_couch_key(self.domain, user_id=user.get('user_id'))
data = XFormInstance.view("reports_forms/all_forms",
startkey=key+[{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if data:
last_seen = util.format_relative_date(data.received_on)
if data.version != '1':
build_id = data.version
else:
build_id = UNKNOWN
form_data = data.get_form
try:
app_name = form_data['meta']['appVersion']['#text']
except KeyError:
try:
app = Application.get(data.app_id)
is_unknown = False
if selected_app and selected_app != data.app_id:
continue
app_name = "%s [%s]" % (app.name, build_id)
except Exception:
app_name = UNKNOWN
if is_unknown and selected_app:
continue
row = [user.get('username_in_report'), last_seen, app_name]
rows.append(row)
return rows
def default_datespan(self):
datespan = super(FormExportReportBase, self).default_datespan
def extract_date(x):
try:
def clip_timezone(datestring):
return datestring[:len('yyyy-mm-ddThh:mm:ss')]
return string_to_datetime(clip_timezone(x['key'][2]))
except Exception:
logging.error("Tried to get a date from this, but it didn't work: %r" % x)
return None
key = make_form_couch_key(self.domain)
startdate = get_db().view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
limit=1,
descending=False,
reduce=False,
wrapper=extract_date
).one()
if startdate:
datespan.startdate = startdate
return datespan
def query(limit=100, userID=None, group=False, username__exclude=["demo_user", "admin"], **kwargs):
if group:
if userID is None:
key = [domain]
else:
key = [domain, userID]
subs = [dict(
userID = r['key'][1],
username = r['key'][2],
deviceID = r['key'][3],
submissions = r['value'],
) for r in get_db().view('cleanup/submissions',
startkey = key,
endkey = key + [{}],
group = True,
)]
total = len(subs)
else:
if userID is None:
key = make_form_couch_key(domain)
subs = XFormInstance.view('reports_forms/all_forms',
startkey=key+[{}],
endkey=key,
reduce=False,
include_docs=True,
descending=True,
limit=limit
)
total = get_db().view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
group_level=1
).one()
total = total['value'] if total else 0
else:
key = make_form_couch_key(domain, user_id=userID)
subs = XFormInstance.view('reports_forms/all_forms',
startkey=key+[{}],
endkey=key,
reduce=False,
include_docs=True,
descending=True,
limit=limit
)
total = get_db().view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
group_level=2
).one()
total = total['value'] if total else 0
_subs = []
for s in subs:
try:
_subs.append(dict(
username = s['form']['meta']['username'],
userID = s['form']['meta']['userID'],
received_on = unicode(s['received_on']),
deviceID = s['form']['meta']['deviceID'],
))
except:
continue
subs = _subs
if username__exclude:
username__exclude = set(username__exclude)
subs = filter(lambda sub: sub['username'] not in username__exclude, subs)
return json_response({
"results": subs,
"total": total,
})
