def run():
all_xforms = XFormInstance.view('pactcarehq/all_submits', include_docs=True, limit=100).all()
total = XFormInstance.view('pactcarehq/all_submits').count()
MongoConnect('pact_test')
for idx, f in enumerate(all_xforms):
print "Saving form %s to mongo: %d/%d" % (f._id, idx, total)
json = f.to_json()
#json['id'] = json['_id']
mx = MongoXform(**json)
mx.save()
def test_broken_save(self):
"""
Test that if the second form submission terminates unexpectedly
and the main form isn't saved, then there are no side effects
such as the original having been marked as deprecated.
"""
class BorkDB(object):
"""context manager for making a db's bulk_save temporarily fail"""
def __init__(self, db):
self.old = {}
self.db = db
def __enter__(self):
self.old['bulk_save'] = self.db.bulk_save
self.db.bulk_save = MagicMock(name='bulk_save',
side_effect=RequestFailed())
def __exit__(self, exc_type, exc_val, exc_tb):
self.db.bulk_save = self.old['bulk_save']
self.assertEqual(
XFormInstance.view('couchforms/edits', key=self.ID).count(), 0)
self.assertFalse(XFormInstance.get_db().doc_exist(self.ID))
xml_data1, xml_data2 = self._get_files()
submit_form_locally(xml_data1, self.domain)
doc = XFormInstance.get(self.ID)
self.assertEqual(self.ID, doc.get_id)
self.assertEqual("XFormInstance", doc.doc_type)
self.assertEqual(self.domain, doc.domain)
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
0
)
with BorkDB(XFormInstance.get_db()):
with self.assertRaises(RequestFailed):
submit_form_locally(xml_data2, self.domain)
# it didn't go through, so make sure there are no edits still
self.assertEqual(
XFormInstance.view('couchforms/edits', key=self.ID).count(), 0)
self.assertTrue(XFormInstance.get_db().doc_exist(self.ID))
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID,
saved=False).count(),
1
)
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
1
)
def testSignal(self):
"""
Test to ensure that with a DOT submission the signal works
"""
start_dot = len(
XFormInstance.view('reports_forms/all_forms',
startkey=[
'submission xmlns', self.domain.name,
XMLNS_DOTS_FORM
],
endkey=[
'submission xmlns', self.domain.name,
XMLNS_DOTS_FORM, {}
],
reduce=False).all())
start_update = len(
XFormInstance.view('reports_forms/all_forms',
startkey=[
'submission xmlns', self.domain.name,
XMLNS_PATIENT_UPDATE_DOT
],
endkey=[
'submission xmlns', self.domain.name,
XMLNS_PATIENT_UPDATE_DOT, {}
],
reduce=False).all())
submit_xform(self.submit_url, self.domain.name, self.pillbox_form)
submitted = XFormInstance.get(PILLBOX_ID)
self.assertTrue(hasattr(submitted, PACT_DOTS_DATA_PROPERTY))
dot_count = XFormInstance.view(
'reports_forms/all_forms',
startkey=['submission xmlns', self.domain.name, XMLNS_DOTS_FORM],
endkey=['submission xmlns', self.domain.name, XMLNS_DOTS_FORM, {}],
).all()[0]['value']
update_count = XFormInstance.view(
'reports_forms/all_forms',
startkey=[
'submission xmlns', self.domain.name, XMLNS_PATIENT_UPDATE_DOT
],
endkey=[
'submission xmlns', self.domain.name, XMLNS_PATIENT_UPDATE_DOT,
{}
],
).all()[0]['value']
self.assertEquals(dot_count, update_count)
self.assertEquals(start_dot + start_update + 2,
dot_count + update_count)
casedoc = CommCareCase.get(CASE_ID)
self.assertEqual(casedoc.xform_ids[-2], PILLBOX_ID)
computed_submit = XFormInstance.get(casedoc.xform_ids[-1])
self.assertEqual(computed_submit.xmlns, XMLNS_PATIENT_UPDATE_DOT)
def process_user(self, user):
username = user.raw_username
# initialize
if username not in self.individual:
self.emails.append(user.email)
self.individual[username] = {
'strategy': self.schedule,
'game': self.schedule,
}
# process this week's forms
key = make_form_couch_key(user.domain, user_id=user.user_id, xmlns=DAILY_DATA_XMLNS)
forms = XFormInstance.view(
"reports_forms/all_forms",
startkey=key + [str(self.week[0])],
endkey=key + [str(self.week[-1] + datetime.timedelta(days=1))],
reduce=False,
include_docs=True
).all()
for form in forms:
self.process_form(form, username)
# get and extend weekly totals
try:
weekly_totals = self.last_week.individual[username]['weekly_totals']
except (LookupError, AttributeError):
weekly_totals = []
weekly_totals.append([
self.week[0].strftime(DATE_FORMAT),
sum([d for d in self.individual[username]['strategy'] if d>0])
])
self.individual[username]['weekly_totals'] = weekly_totals
def _get_submissions_for_patient_by_date(patient, visit_dates, schema='http://dev.commcarehq.org/pact/dots_form'):
"""Argument: Patient django object, visit date
Will return a view result of all submissions by patient where the key is the patient pact_id
return value: [pact_id, year, month, day]=>submission"""
keys = []
date_key_map = {}
#t2 = datetime.now()
for visit_date in visit_dates:
day_of_week = visit_date.isoweekday()-1
yearstart = visit_date.year
monthstart = visit_date.month
datestart = visit_date.day
#get the xform count for that day
key = [patient.couchdoc.pact_id, yearstart, monthstart, datestart, schema]
keys.append(key)
key_str = ''.join([str(x) for x in key])
date_key_map[key_str] = visit_date
submit_reduction = XFormInstance.view('pactcarehq/all_submits_by_patient_date', keys=keys)
#d2 = datetime.now()-t2
#print "\tSingle Patient data query QUERY: %d.%d" % (d2.seconds, d2.microseconds/1000)
#t3 = datetime.now()
ret = {} #a return value of date ordered submissions by
for row in submit_reduction:
key = row['key']
key_str = ''.join([str(x) for x in key])
submits = row['value']
date = date_key_map[key_str]
ret[date] = [XFormInstance.wrap(x) for x in submits]
#d3 = datetime.now()-t3
#print "\tSingle Patient data query HASHING: %d.%d" % (d3.seconds, d3.microseconds/1000)
return ret
def handle(self, *args, **options):
key = make_form_couch_key("hsph")
test_forms = XFormInstance.view('reports_forms/all_forms',
reduce=False,
startkey=key,
endkey=key+["2012-11-07"],
include_docs=True,
).all()
test_cases = CommCareCase.view('reports/case_activity',
reduce=False,
startkey=["","hsph"],
endkey=["","hsph","2012-11-07"],
include_docs=True,
).all()
print "\nDELETING TEST HSPH FORMS"
for form in test_forms:
if form.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
form.delete()
print "\n\nDELETING TEST HSPH CASES"
for case in test_cases:
if case.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
case.delete()
print "\n"
def get_commtrack_forms(self, domain):
return XFormInstance.view(
'reports_forms/all_forms',
startkey=['submission xmlns', domain, COMMTRACK_REPORT_XMLNS],
endkey=['submission xmlns', domain, COMMTRACK_REPORT_XMLNS, {}],
reduce=False,
include_docs=True)
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage: manage.py submit_forms <domain>')
domain = args[0]
key = make_form_couch_key(domain)
submissions = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key + [{}],
include_docs=True)
ids_by_username = dict()
def get_id(username):
if username in ids_by_username:
return ids_by_username[username]
else:
userID = get_db().view('users/logins_by_username',
key=username).one()
userID = userID['value'] if userID else None
ids_by_username[username] = userID
return userID
for submission in submissions:
if 'meta' in submission.form:
username = format_username(submission.form['meta']['username'],
domain)
userID = get_id(username)
if userID:
submission.form['meta']['userID'] = userID
submission.save()
print submission._id, username, userID
else:
print "skipped: %s" % submission._id
def get_commtrack_forms(self, domain):
return XFormInstance.view('reports_forms/all_forms',
startkey=['submission xmlns', domain, COMMTRACK_REPORT_XMLNS],
endkey=['submission xmlns', domain, COMMTRACK_REPORT_XMLNS, {}],
reduce=False,
include_docs=True
)
def report_context(self):
ward = self.request.GET.get("group", None)
month = self.request.GET.get("month", None)
year = self.request.GET.get("year", None)
if not (ward and month and year):
return dict()
ward = Group.get(ward)
user_ids = get_db().view('pathfinder/pathfinder_gov_reg_by_username',
keys=[[self.domain, w] for w in ward.users],
include_docs=True).all()
chws = QueryableList(map(CommCareUser.get, ward.users))
chws._reported = lambda x: x['reported_this_month']
for c in chws:
r = XFormInstance.view(
'pathfinder/pathfinder_xforms',
key=['pathfinder', c.get_id,
int(year), int(month)],
reduce=True).one()
c['reported_this_month'] = (r != None)
return dict(p=retrieve_patient_group(user_ids, self.domain, year,
month),
chws=chws,
ward=ward,
date=date(year=int(year), month=int(month), day=01))
def process_user(self, user):
username = user.raw_username
# initialize
if username not in self.individual:
self.emails.append(user.email)
self.individual[username] = {
'strategy': self.schedule,
'game': self.schedule,
}
# process this week's forms
key = make_form_couch_key(user.domain, user_id=user.user_id, xmlns=DAILY_DATA_XMLNS)
forms = XFormInstance.view(
"all_forms/view",
startkey=key + [str(self.week[0])],
endkey=key + [str(self.week[-1] + datetime.timedelta(days=1))],
reduce=False,
include_docs=True
).all()
for form in forms:
self.process_form(form, username)
# get and extend weekly totals
try:
weekly_totals = self.last_week.individual[username]['weekly_totals']
except (LookupError, AttributeError):
weekly_totals = []
weekly_totals.append([
self.week[0].strftime(DATE_FORMAT),
sum([d for d in self.individual[username]['strategy'] if d>0])
])
self.individual[username]['weekly_totals'] = weekly_totals
def delete_all_data(request, domain, template="cleanup/delete_all_data.html"):
if request.method == 'GET':
return render(request, template, {
'domain': domain
})
key = make_form_couch_key(domain)
xforms = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
include_docs=True,
reduce=False
)
cases = CommCareCase.view('case/by_date_modified',
startkey=[domain, {}, {}],
endkey=[domain, {}, {}, {}],
include_docs=True,
reduce=False
)
suffix = DELETED_SUFFIX
deletion_id = random_hex()
for thing_list in (xforms, cases):
for thing in thing_list:
thing.doc_type += suffix
thing['-deletion_id'] = deletion_id
thing.save()
return HttpResponseRedirect(reverse('homepage'))
def xml_download(request):
username = request.user.username
offset =0
limit_count=100
temp_xml = tempfile.TemporaryFile()
temp_xml.write("<restoredata>\n")
total_count = 0
db = get_db()
xforms = XFormInstance.view("pactcarehq/all_submits", key=username).all()
for form in xforms:
try:
xml_str = db.fetch_attachment(form['id'], 'form.xml').replace("<?xml version=\'1.0\' ?>", '')
temp_xml.write(xml_str)
temp_xml.write("\n")
total_count += 1
except ResourceNotFound:
logging.error("Error, xform submission %s does not have a form.xml attachment." % (form._id))
print total_count
temp_xml.write("</restoredata>")
length = temp_xml.tell()
temp_xml.seek(0)
wrapper = FileWrapper(temp_xml)
response = HttpResponse(wrapper, mimetype='text/xml')
response['Content-Length'] = length
return response
def _get_submissions_for_patient(patient):
"""Returns a view of all the patients submissions by the patient's case_id (which is their PactPatient doc_id, this probably should be altered)
params: patient=Patient (django) object
returns: array of XFormInstances for a given patient.
"""
#xform_submissions = XFormInstance.view('pactcarehq/all_submits_by_case', key=patient.doc_id, include_docs=True)
xform_submissions = XFormInstance.view('pactcarehq/all_submits_by_patient_date', startkey=[patient.couchdoc.pact_id, 0000], endkey=[patient.couchdoc.pact_id, 9999], include_docs=True)
submissions = []
for note in xform_submissions:
if not note.form.has_key('case'):
continue
xmlns = note['xmlns']
if xmlns == 'http://dev.commcarehq.org/pact/dots_form':
formtype = "DOTS"
date = note['form']['encounter_date']
elif xmlns == "http://dev.commcarehq.org/pact/progress_note":
formtype = "Progress Note"
date = note['form']['note']['encounter_date']
#commenting out patientupdate until submissions get fixed, and handling is done correctly for case data
# elif xmlns == "http://dev.commcarehq.org/pact/patientupdate":
# formtype = "Patient Update"
# date = note['form']
else:
logging.debug("Skipping these namespaces until they are handled correctly %s" % (xmlns))
continue
submissions.append([note._id, date, note.form['Meta']['username'] , formtype])
submissions=sorted(submissions, key=lambda x: x[1], reverse=True)
return submissions
def report_context(self):
ward = self.request.GET.get("group", None)
month = self.request.GET.get("month", None)
year = self.request.GET.get("year", None)
if not (ward and month and year):
return dict()
ward = Group.get(ward)
user_ids = get_db().view('pathfinder/pathfinder_gov_reg_by_username', keys=[[self.domain, w]
for w in ward.users], include_docs=True).all()
chws = QueryableList(map(CommCareUser.get, ward.users))
chws._reported = lambda x: x['reported_this_month']
for c in chws:
r = XFormInstance.view('pathfinder/pathfinder_xforms',
key=['pathfinder', c.get_id, int(year), int(month)],
reduce=True
).one()
c['reported_this_month'] = (r != None)
return dict(
p=retrieve_patient_group(user_ids, self.domain, year, month),
chws=chws,
ward=ward,
date=date(year=int(year),month=int(month), day=01)
)
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage: manage.py submit_forms <domain>')
domain = args[0]
key = make_form_couch_key(domain)
submissions = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
include_docs=True
)
ids_by_username = dict()
def get_id(username):
if username in ids_by_username:
return ids_by_username[username]
else:
userID = get_db().view('users/logins_by_username', key=username).one()
userID = userID['value'] if userID else None
ids_by_username[username] = userID
return userID
for submission in submissions:
if 'meta' in submission.form:
username = format_username(submission.form['meta']['username'], domain)
userID = get_id(username)
if userID:
submission.form['meta']['userID'] = userID
submission.save()
print submission._id, username, userID
else:
print "skipped: %s" % submission._id
def handle(self, *args, **options):
key = make_form_couch_key("hsph")
test_forms = XFormInstance.view(
'reports_forms/all_forms',
reduce=False,
startkey=key,
endkey=key + ["2012-11-07"],
include_docs=True,
).all()
test_cases = CommCareCase.view(
'reports/case_activity',
reduce=False,
startkey=["", "hsph"],
endkey=["", "hsph", "2012-11-07"],
include_docs=True,
).all()
print "\nDELETING TEST HSPH FORMS"
for form in test_forms:
if form.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
form.delete()
print "\n\nDELETING TEST HSPH CASES"
for case in test_cases:
if case.domain == "hsph":
sys.stdout.write(".")
sys.stdout.flush()
case.delete()
def get_commtrack_forms(domain):
key = ['submission xmlns', domain, COMMTRACK_REPORT_XMLNS]
return XFormInstance.view('all_forms/view',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=True)
def all_forms_without_domains():
return XFormInstance.view(
'reports_forms/all_forms',
reduce=False,
startkey=['completion', None],
endkey=["completion", None, {}],
include_docs=True,
)
def tearDown(self):
for xform in XFormInstance.view(
'reports_forms/all_forms',
startkey=['submission', TEST_DOMAIN],
endkey=['submission', TEST_DOMAIN, {}],
reduce=False,
include_docs=True,
):
xform.delete()
def get_all_forms_in_all_domains():
assert settings.UNIT_TESTING, (
'You can only call {} when unit testing'.format(
get_all_forms_in_all_domains.__name__))
return XFormInstance.view(
'hqadmin/forms_over_time',
reduce=False,
include_docs=True,
).all()
def get_commtrack_forms(domain):
key = ['submission xmlns', domain, COMMTRACK_REPORT_XMLNS]
return XFormInstance.view(
'all_forms/view',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=True
)
def get_last_form_submission(self):
form = XFormInstance.view("couchforms/all_submissions_by_domain",
startkey=[self.domain, "by_type", "XFormInstance", {}],
endkey=[self.domain, "by_type", "XFormInstance"],
descending=True,
include_docs=True,
reduce=False,
).first()
return form
def _get_submissions(domain, keys):
xforms = []
for key in keys:
for sub in XFormInstance.view('cleanup/submissions', reduce=False,
key=[domain, key['userID'], key['username'], key['deviceID']],
include_docs=True
).all():
xforms.append(sub)
return xforms
def _forms_by_domain(domain):
res = XFormInstance.view(
'couchforms/all_submissions_by_domain',
startkey=[domain, "by_type", "XFormInstance"],
endkey=[domain, "by_type", "XFormInstance", {}],
reduce=True,
include_docs=False,
).all()
return res[0].get('value', 0) if res else 0
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationFilter.slug,
'')
for user in self.users:
last_seen = last_sync = app_name = None
key = make_form_couch_key(self.domain,
user_id=user.user_id,
app_id=selected_app or None)
xform = XFormInstance.view(
"reports_forms/all_forms",
startkey=key + [{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if xform:
last_seen = xform.received_on
build_version, build_version_source = get_build_version(xform)
if xform.app_id:
try:
app = get_app(self.domain, xform.app_id)
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform)
build_html = _build_html(build_version, build_version_source)
app_name = app_name or _("Unknown App")
app_name = format_html(
u'{} {}',
app_name,
mark_safe(build_html),
)
if app_name is None and selected_app:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append([
user.username_in_report,
_fmt_date(last_seen),
_fmt_date(last_sync), app_name or "---"
])
return rows
def get_user_latest_xform(self, user):
return XFormInstance.view(
'all_forms/view',
startkey=['submission xmlns user', self.domain, COMMTRACK_REPORT_XMLNS, user.get_id, {}],
endkey=['submission xmlns user', self.domain, COMMTRACK_REPORT_XMLNS, user.get_id],
reduce=False,
include_docs=True,
descending=True,
limit=1
).first()
def form_count(user_id):
key = make_form_couch_key(self.domain, user_id=user_id)
result = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key + [{}],
group_level=0).one()
if result:
return result['value']
else:
return 0
def get_all_forms_in_all_domains():
assert settings.UNIT_TESTING, (
'You can only call {} when unit testing'
.format(get_all_forms_in_all_domains.__name__)
)
return XFormInstance.view(
'hqadmin/forms_over_time',
reduce=False,
include_docs=True,
).all()
def setUp(self):
for item in XFormInstance.view("couchforms/by_xmlns", include_docs=True, reduce=False).all():
item.delete()
file_path = os.path.join(os.path.dirname(__file__), "data", "meta.xml")
with open(file_path, "rb") as f:
xml_data = f.read()
self.instance = post_xform_to_couch(xml_data)
def get_number_of_forms_of_all_types(domain):
startkey = [domain]
endkey = startkey + [{}]
submissions = XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=True,
).one()
return submissions['value'] if submissions else 0
def all_submissions(self):
if self._all_submissions is None:
self._all_submissions = XFormInstance.view(
"reports/all_submissions",
startkey=[self.domain, self.datespan.startdate_param_utc],
endkey=[self.domain, self.datespan.enddate_param_utc],
include_docs=True,
reduce=False,
)
return self._all_submissions
def form_count(user_id):
result = XFormInstance.view('couchforms/by_user',
startkey=[user_id],
endkey=[user_id, {}],
group_level=0
).one()
if result:
return result['value']
else:
return 0
def get_user_latest_xform(self, user):
return XFormInstance.view(
'all_forms/view',
startkey=['submission xmlns user', self.domain, COMMTRACK_REPORT_XMLNS, user.get_id, {}],
endkey=['submission xmlns user', self.domain, COMMTRACK_REPORT_XMLNS, user.get_id],
reduce=False,
include_docs=True,
descending=True,
limit=1
).first()
def get_number_of_forms_of_all_types(domain):
startkey = [domain]
endkey = startkey + [{}]
submissions = XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=True,
).one()
return submissions['value'] if submissions else 0
def setUp(self):
self.domain = Domain.get_or_create_with_name(TEST_DOMAIN, is_active=True)
self.user = WebUser.create(TEST_DOMAIN, TEST_USER, TEST_PASSWORD)
self.user.set_role(self.domain.name, 'admin')
self.user.save()
for item in CommCareCase.view("case/by_user", include_docs=True, reduce=False).all():
item.delete()
for item in XFormInstance.view("hqadmin/forms_over_time", include_docs=True, reduce=False).all():
item.delete()
time.sleep(1)
def _household_verification_json(
domain="dodoma",
last_hvid_path=["household_verification"],
next_hvid_path=["followup_id"],
xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79',
range=None):
if range:
start, end = map(string_to_datetime, range)
else:
now = datetime.utcnow()
start, end = now - timedelta(days=7), now
key = make_form_couch_key(domain, xmlns=xmlns)
submissions = XFormInstance.view(
'reports_forms/all_forms',
reduce=False,
startkey=key + [json_format_datetime(start)],
endkey=key + [json_format_datetime(end)],
include_docs=True,
)
stats = get_household_verification_data(
submissions=submissions,
next_hvid_path=next_hvid_path,
last_hvid_path=last_hvid_path,
)
stats_by_userID = {}
for s in stats:
stats_by_userID[s['userID']] = s
s['username'] = "******" % s['userID']
users = CommCareUser.by_domain(domain)
for user in users:
userID = user.user_id
username = user_id_to_username(userID)
if userID in stats_by_userID:
stats_by_userID[userID]['username'] = username
else:
stats.append({
'userID': userID,
'username': username,
'total': 0,
'correct': 0
})
stats.sort(key=lambda s: s['username'])
return {
"headers": ["Username", "Correct", "Total", "Percent Correct"],
"rows": [[
s['username'], s['correct'], s['total'],
("%s%%" %
int(s['correct'] * 100 / s['total']) if s['total'] else "---")
] for s in stats],
}
def _get_all_form_submissions(domain, startdate=None, enddate=None):
key = make_form_couch_key(domain)
startkey = key+[startdate] if startdate and enddate else key
endkey = key+[enddate] if startdate and enddate else key + [{}]
submissions = XFormInstance.view('reports_forms/all_forms',
startkey=startkey,
endkey=endkey,
include_docs=True,
reduce=False
)
return submissions
def get_number_of_forms_by_type(domain, type_):
assert type_ in doc_types()
startkey = [domain, type_]
endkey = startkey + [{}]
submissions = XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=True,
).one()
return submissions['value'] if submissions else 0
def get_number_of_forms_by_type(domain, type_):
assert type_ in doc_types()
startkey = [domain, type_]
endkey = startkey + [{}]
submissions = XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=True,
).one()
return submissions['value'] if submissions else 0
def form_count(user_id):
key = make_form_couch_key(self.domain, user_id=user_id)
result = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key + [{}],
group_level=0
).one()
if result:
return result['value']
else:
return 0
def get_number_of_forms_in_all_domains():
"""
Return number of non-error forms (but incl. logs) total across all domains
specifically as stored in couch.
(Can't rewrite to pull from ES or SQL; this function is used as a point
of comparison between row counts in other stores.)
"""
return XFormInstance.view('hqadmin/forms_over_time').one()['value']
def get_forms_of_all_types(domain):
startkey = [domain]
endkey = startkey + [{}]
return XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
include_docs=True,
classes=doc_types(),
).all()
def get_forms_of_all_types(domain):
assert settings.UNIT_TESTING
startkey = [domain]
endkey = startkey + [{}]
return XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
include_docs=True,
classes=doc_types(),
).all()
def testSignal(self):
"""
Test to ensure that with a DOT submission the signal works
"""
start_dot = len(XFormInstance.view('couchforms/by_xmlns', key=XMLNS_DOTS_FORM, reduce=False).all())
start_update = len(XFormInstance.view('couchforms/by_xmlns', key=XMLNS_PATIENT_UPDATE_DOT, reduce=False).all())
submit_xform(self.submit_url, self.domain.name, self.pillbox_form)
submitted = XFormInstance.get(PILLBOX_ID)
self.assertTrue(hasattr(submitted, PACT_DOTS_DATA_PROPERTY))
dot_count = XFormInstance.view('couchforms/by_xmlns', key=XMLNS_DOTS_FORM).all()[0]['value']
update_count = XFormInstance.view('couchforms/by_xmlns', key=XMLNS_PATIENT_UPDATE_DOT).all()[0]['value']
self.assertEquals(dot_count, update_count)
self.assertEquals(start_dot+start_update + 2, dot_count + update_count)
casedoc = CommCareCase.get(CASE_ID)
self.assertEqual(casedoc.xform_ids[-2], PILLBOX_ID)
computed_submit = XFormInstance.get(casedoc.xform_ids[-1])
self.assertEqual(computed_submit.xmlns, XMLNS_PATIENT_UPDATE_DOT)
def _household_verification_json(
domain="dodoma",
last_hvid_path=["household_verification"],
next_hvid_path=["followup_id"],
xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79',
range=None
):
if range:
start, end = map(string_to_datetime, range)
else:
now = datetime.utcnow()
start, end = now - timedelta(days=7), now
key = make_form_couch_key(domain, xmlns=xmlns)
submissions = XFormInstance.view('reports_forms/all_forms',
reduce=False,
startkey=key+[json_format_datetime(start)],
endkey=key+[json_format_datetime(end)],
include_docs=True,
)
stats = get_household_verification_data(
submissions=submissions,
next_hvid_path=next_hvid_path,
last_hvid_path=last_hvid_path,
)
stats_by_userID = {}
for s in stats:
stats_by_userID[s['userID']] = s
s['username'] = "******" % s['userID']
users = CommCareUser.by_domain(domain)
for user in users:
userID = user.user_id
username = user_id_to_username(userID)
if userID in stats_by_userID:
stats_by_userID[userID]['username'] = username
else:
stats.append({'userID': userID, 'username': username, 'total': 0, 'correct': 0})
stats.sort(key=lambda s: s['username'])
return {
"headers": ["Username", "Correct", "Total", "Percent Correct"],
"rows": [[
s['username'],
s['correct'],
s['total'],
("%s%%" % int(s['correct']*100/s['total']) if s['total'] else "---")
] for s in stats],
}
def get_reports(cls, domain, location=None, datespan=None):
start = datespan.startdate if datespan else datetime(1900, 1, 1)
end = datespan.end_of_end_day if datespan else datetime.max
timestamp_start = dateparse.json_format_datetime(start)
timestamp_end = dateparse.json_format_datetime(end)
loc_id = location._id if location else None
startkey = [domain, loc_id, timestamp_start]
endkey = [domain, loc_id, timestamp_end]
return [StockReport(f) for f in \
XFormInstance.view('commtrack/stock_reports',
startkey=startkey,
endkey=endkey,
include_docs=True)]
def get_days_on(date):
week = get_m_to_f(date)
start = week[0] - datetime.timedelta(days=1)
end = week[-1] + datetime.timedelta(days=1)
forms = XFormInstance.view(
'reports_forms/all_forms',
startkey=['submission xmlns', DOMAIN, WEEKLY_SCHEDULE_XMLNS, start.isoformat()],
endkey=['submission xmlns', DOMAIN, WEEKLY_SCHEDULE_XMLNS, end.isoformat()],
reduce=False,
include_docs=True,
).all()
if forms:
forms.sort(key=lambda form: form.received_on)
return forms[-1].form
def couch_check():
"""check couch"""
#in reality when things go wrong with couch and postgres (as of this
# writing) - it's far from graceful, so this will # likely never be
# reached because another exception will fire first - but for
# completeness sake, this check is done here to verify our calls will
# work, and if other error handling allows the request to get this far.
try:
xforms = XFormInstance.view('reports_forms/all_forms', limit=1).all()
except:
xforms = None
return (isinstance(xforms, list), None)
def setUp(self):
self.domain = Domain.get_or_create_with_name(TEST_DOMAIN,
is_active=True)
self.user = WebUser.create(TEST_DOMAIN, TEST_USER, TEST_PASSWORD)
self.user.set_role(self.domain.name, 'admin')
self.user.save()
for item in CommCareCase.view("case/by_user",
include_docs=True,
reduce=False).all():
item.delete()
for item in XFormInstance.view("hqadmin/forms_over_time",
include_docs=True,
reduce=False).all():
item.delete()
time.sleep(1)
def populate_dup_map(self, domain):
self.dups_by_domain[domain] = defaultdict(set)
dups = XFormInstance.view('couchforms/all_submissions_by_domain',
startkey=[domain, 'XFormDuplicate', '2016'],
endkey=[domain, "XFormDuplicate", {}],
reduce=False,
include_docs=True)
for dup in dups:
match = re.match('Form is a duplicate of another! \((.*)\)',
dup.problem or "")
if match:
orig_id = match.groups()[0]
try:
orig_id = UUID(orig_id)
except ValueError:
continue
self.dups_by_domain[domain][orig_id].add(dup._id)
def get_forms_by_type(domain, type_, recent_first=False, limit=None):
assert type_ in doc_types()
# no production code should be pulling all forms in one go!
assert limit is not None
startkey = [domain, type_]
endkey = startkey + [{}]
if recent_first:
startkey, endkey = endkey, startkey
return XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
descending=recent_first,
include_docs=True,
limit=limit,
classes=doc_types(),
).all()
def update_indicators_for_xmlns(self, domain, form_label_filter=None):
key = [MVP.NAMESPACE, domain]
all_labels = FormLabelIndicatorDefinition.get_db().view(
'indicators/form_labels',
reduce=False,
startkey=key,
endkey=key + [{}],
).all()
for label in all_labels:
label_name = label['value']
if form_label_filter is not None and form_label_filter != label_name:
continue
xmlns = label['key'][-2]
print("\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain))
relevant_forms = XFormInstance.get_db().view(
"all_forms/view",
reduce=True,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).first()
num_forms = relevant_forms['value'] if relevant_forms else 0
form_ids = [r['id'] for r in XFormInstance.view(
"all_forms/view",
reduce=False,
include_docs=False,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).all()]
print("Found %d forms with matching XMLNS %s" % (num_forms, xmlns))
relevant_indicators = FormIndicatorDefinition.get_all(
namespace=MVP.NAMESPACE,
domain=domain,
xmlns=xmlns
)
if relevant_indicators:
self._throttle_updates(
"Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (
label_name, xmlns, domain),
relevant_indicators, num_forms, domain,
form_ids, XFormInstance)
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationField.slug, '')
UNKNOWN = _("unknown")
for user in self.users:
last_seen = self.table_cell(-1, _("Never"))
app_name = "---"
is_unknown = True
key = make_form_couch_key(self.domain, user_id=user.get('user_id'))
data = XFormInstance.view("reports_forms/all_forms",
startkey=key+[{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if data:
last_seen = util.format_relative_date(data.received_on)
if data.version != '1':
build_id = data.version
else:
build_id = UNKNOWN
form_data = data.get_form
try:
app_name = form_data['meta']['appVersion']['#text']
except KeyError:
try:
app = Application.get(data.app_id)
is_unknown = False
if selected_app and selected_app != data.app_id:
continue
app_name = "%s [%s]" % (app.name, build_id)
except Exception:
app_name = UNKNOWN
if is_unknown and selected_app:
continue
row = [user.get('username_in_report'), last_seen, app_name]
rows.append(row)
return rows
def change_submissions_app_id(request, domain):
app_id = request.POST['app_id'] or None
xmlns = request.POST['xmlns']
new_app_id = request.POST['new_app_id']
next = request.POST['next']
submissions = XFormInstance.view('exports_forms/by_xmlns',
key=['^XFormInstance', domain, app_id, xmlns],
include_docs=True,
reduce=False,
).all()
for sub in submissions:
assert(getattr(sub, 'app_id', None) == app_id)
assert(sub.xmlns == xmlns)
sub.app_id = new_app_id
sub.save()
messages.success(request, 'Your fix was successful and affected %s submissions' % len(submissions))
return HttpResponseRedirect(next)
def setUp(self):
for item in XFormInstance.view("hqadmin/forms_over_time", include_docs=True, reduce=False).all():
item.delete()
for item in FormData.objects.all():
item.delete()
file_path = os.path.join(os.path.dirname(__file__), "data", "meta.xml")
with open(file_path, "rb") as f:
xml_data = f.read()
SubmissionPost(
instance=xml_data,
domain='sofabed',
app_id='12345',
received_on=datetime.now()
).get_response()
self.instance = XFormInstance.get('THIS_IS_THE_INSTANCEID')
def export_raw_data(export_tag, filename=None):
# really this shouldn't be here, but keeping it for now
from couchforms.models import XFormInstance
xform_instances = XFormInstance.view(
'couchexport/schema_index',
include_docs=True,
reduce=False,
**get_schema_index_view_keys(export_tag))
f = StringIO()
zipfile = ZipFile(f, 'w')
for xform_instance in xform_instances:
form_xml = xform_instance.fetch_attachment('form.xml').encode('utf-8')
zipfile.writestr("%s.xml" % xform_instance.get_id, form_xml)
zipfile.close()
f.flush()
response = HttpResponse(f.getvalue())
f.close()
response['Content-Type'] = "application/zip"
response['Content-Disposition'] = "attachment; filename=%s.zip" % filename
return response
