def tearDown(self):
self.case_repeater.delete()
self.form_repeater.delete()
XFormInstance.get(instance_id).delete()
repeat_records = RepeatRecord.all()
for repeat_record in repeat_records:
repeat_record.delete()
def save_processed_models(cls, processed_forms, cases=None, stock_updates=None):
docs = list(processed_forms) + (cases or [])
docs = filter(None, docs)
assert XFormInstance.get_db().uri == CommCareCase.get_db().uri
XFormInstance.get_db().bulk_save(docs)
for stock_update in stock_updates or []:
stock_update.commit()
def tag_forms_as_deleted_rebuild_associated_cases(user_id, domain, form_id_list, deletion_id,
deletion_date, deleted_cases=None):
"""
Upon user deletion, mark associated forms as deleted and prep cases
for a rebuild.
- 2 saves/sec for cloudant slowness (rate_limit)
"""
if deleted_cases is None:
deleted_cases = set()
cases_to_rebuild = set()
forms_to_check = get_docs(XFormInstance.get_db(), form_id_list)
forms_to_save = []
for form in forms_to_check:
assert form['domain'] == domain
if not is_deleted(form):
form['doc_type'] += DELETED_SUFFIX
form['-deletion_id'] = deletion_id
form['-deletion_date'] = deletion_date
forms_to_save.append(form)
# rebuild all cases anyways since we don't know if this has run or not if the task was killed
cases_to_rebuild.update(get_case_ids_from_form(form))
XFormInstance.get_db().bulk_save(forms_to_save)
detail = UserArchivedRebuild(user_id=user_id)
for case in cases_to_rebuild - deleted_cases:
_rebuild_case_with_retries.delay(domain, case, detail)
def setUp(self):
for form in get_docs([DOMAIN, "http://www.commcarehq.org/export/test"]):
XFormInstance.wrap(form).delete()
dom = create_domain(DOMAIN)
self.couch_user = WebUser.create(None, "test", "foobar")
self.couch_user.add_domain_membership(DOMAIN, is_admin=True)
self.couch_user.save()
def growth_monitoring_forms_context():
forms_data = [
{
'child_first_name': 'Foo',
'dhis2_te_inst_id': '', # Not enrolled
'dhis2_processed': '' # Not processed
},
{
'child_first_name': 'Bar',
'dhis2_te_inst_id': '123', # Enrolled
'dhis2_processed': '' # Not processed
},
{
'child_first_name': 'Baz',
'dhis2_te_inst_id': '456', # Enrolled
'dhis2_processed': '' # Not processed
}
]
forms = []
for data in forms_data:
form = XFormInstance(domain=DOMAIN, form=data)
form.save()
forms.append(form)
yield forms
def handle(self, *args, **options):
if len(args) < 2:
print "please specify a filepath and an archiving_user"
return
filepath = args[0]
archiving_user = args[1]
try:
form_ids = open(filepath)
except Exception as e:
print "there was an issue opening the file: %s" % e
return
try:
form_ids = [f[0] for f in csv.reader(form_ids)]
except Exception as e:
print "there was an issue reading the file %s" % e
return
for xform_doc in iter_docs(XFormInstance.get_db(), form_ids):
try:
xform = XFormInstance.wrap(xform_doc)
xform.archive(user_id=archiving_user)
print "Archived form %s in domain %s" % (
xform._id, xform.domain
)
except Exception as e:
print "Issue archiving XFORM %s for domain %s: %s" % (
xform_doc['_id'], xform_doc['domain'], e
)
def test_simple_swap(self):
# Test a form with a single dup
bad_form_id = self._submit_form()._id
good_dup_id = self._submit_form(bad_form_id)._id
with tempdir() as tmp:
ids_file_path = os.path.join(tmp, 'ids')
with open(ids_file_path, "w") as ids_file:
ids_file.write("{} {}\n".format(DOMAIN, bad_form_id))
call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)
# Throw in a second call to the script to test idempotence as well
call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)
bad_form = XFormInstance.get(bad_form_id)
self.assertEqual(bad_form.doc_type, "XFormDuplicate")
self.assertRegexpMatches(
bad_form.problem, BAD_FORM_PROBLEM_TEMPLATE.format(good_dup_id, "")
)
good_dup_form = XFormInstance.get(good_dup_id)
self.assertEqual(good_dup_form.doc_type, "XFormInstance")
self.assertRegexpMatches(
good_dup_form.problem, FIXED_FORM_PROBLEM_TEMPLATE.format(
id_=bad_form_id, datetime_=""
)
)
def update_analytics_indexes():
"""
Mostly for testing; wait until analytics data sources are up to date
so that calls to analytics functions return up-to-date
"""
XFormInstance.get_db().view("reports_forms/all_forms", limit=1).all()
XFormInstance.get_db().view("exports_forms/by_xmlns", limit=1).all()
def archive_forms(domain, user, uploaded_data):
response = {"errors": [], "success": []}
form_ids = [row.get("form_id") for row in uploaded_data]
missing_forms = set(form_ids)
for xform_doc in iter_docs(XFormInstance.get_db(), form_ids):
xform = XFormInstance.wrap(xform_doc)
missing_forms.discard(xform["_id"])
if xform["domain"] != domain:
response["errors"].append(
_(u"XFORM {form_id} does not belong to domain {domain}").format(
form_id=xform["_id"], domain=xform["domain"]
)
)
continue
xform_string = _(u"XFORM {form_id} for domain {domain} by user '{username}'").format(
form_id=xform["_id"], domain=xform["domain"], username=user.username
)
try:
xform.archive(user=user.username)
response["success"].append(_(u"Successfully archived {form}").format(form=xform_string))
except Exception as e:
response["errors"].append(_(u"Could not archive {form}: {error}").format(form=xform_string, error=e))
for missing_form_id in missing_forms:
response["errors"].append(_(u"Could not find XForm {form_id}").format(form_id=missing_form_id))
return response
def test_basic_edit(self):
xml_data1, xml_data2 = self._get_files()
docs = []
doc = post_xform_to_couch(xml_data1)
self.assertEqual(self.ID, doc.get_id)
self.assertEqual("XFormInstance", doc.doc_type)
self.assertEqual("", doc.form["vitals"]["height"])
self.assertEqual("other", doc.form["assessment"]["categories"])
doc.domain = "test-domain"
doc.save()
doc = post_xform_to_couch(xml_data2, domain="test-domain")
self.assertEqual(self.ID, doc.get_id)
self.assertEqual("XFormInstance", doc.doc_type)
self.assertEqual("100", doc.form["vitals"]["height"])
self.assertEqual("Edited Baby!", doc.form["assessment"]["categories"])
docs.append(doc)
doc = XFormDeprecated.view("couchforms/edits", include_docs=True).first()
self.assertEqual(self.ID, doc.orig_id)
self.assertNotEqual(self.ID, doc.get_id)
self.assertEqual(XFormDeprecated.__name__, doc.doc_type)
self.assertEqual("", doc.form["vitals"]["height"])
self.assertEqual("other", doc.form["assessment"]["categories"])
self.assertEqual(XFormInstance.get_db().fetch_attachment(doc.get_id, "form.xml"), xml_data1)
self.assertEqual(XFormInstance.get_db().fetch_attachment(self.ID, "form.xml"), xml_data2)
for doc in docs:
doc.delete()
def rows(self):
domain = self.report.filter_values['domain']
startdate = self.report.filter_values['startdate']
enddate = self.report.filter_values['enddate']
key_base = 'submission xmlns user'
# todo this will do one couch view hit per relevant user. could be optimized to sql or something if desired
user_ids = self.report.get_user_ids()
rows = []
for user in user_ids:
last_submission = XFormInstance.get_db().view('all_forms/view',
startkey=[key_base, domain, self.xmlns, user, enddate],
endkey=[key_base, domain, self.xmlns, user, startdate],
limit=1,
reduce=False,
include_docs=True,
descending=True,
).one()
if last_submission:
wrapped = XFormInstance.wrap(last_submission['doc'])
user_row = [wrapped.get_data(path) for path in self.column_slugs]
else:
user_row = [NO_VALUE] * len(self.column_slugs)
rows.append((user, user_row))
# format
formatted_rows = list(self.report.format.format_output(rows))
# transpose
return [[_(col)] + [r[i] for r in formatted_rows] for i, col in enumerate(self.column_slugs)]
def process_cases(xform, config=None):
"""
Creates or updates case objects which live outside of the form.
If reconcile is true it will perform an additional step of
reconciling the case update history after the case is processed.
"""
warnings.warn(
'This function is deprecated. You should be using SubmissionPost.',
DeprecationWarning,
)
assert getattr(settings, 'UNIT_TESTING', False)
domain = get_and_check_xform_domain(xform)
with CaseDbCache(domain=domain, lock=True, deleted_ok=True) as case_db:
case_result = process_cases_with_casedb([xform], case_db, config=config)
cases = case_result.cases
docs = [xform] + cases
now = datetime.datetime.utcnow()
for case in cases:
case.server_modified_on = now
XFormInstance.get_db().bulk_save(docs)
for case in cases:
case_post_save.send(CommCareCase, case=case)
case_result.commit_dirtiness_flags()
return cases
def setUpClass(cls):
super(TestCouchDomainFormChangeProvider, cls).setUpClass()
cls.domains = ["d1", "d2", "d3"]
cls.form_ids = {
(domain, doc_type): ["{}-{}-{}".format(doc_type, domain, i) for i in range(3)]
for domain in cls.domains
for doc_type in ["XFormInstance", "XFormArchived"]
}
cls.couch_db = FakeCouchDb(
views={
"by_domain_doc_type_date/view": [
(
{
"startkey": list(domain_doc_type),
"endkey": list(domain_doc_type) + [{}],
"include_docs": True,
"limit": 1000,
"reduce": False,
},
[cls._get_row(form_id, domain_doc_type[0], domain_doc_type[1]) for form_id in form_ids],
)
for domain_doc_type, form_ids in cls.form_ids.items()
]
}
)
XFormInstance.set_db(cls.couch_db)
def test_second_edit_fails(self):
form_id = uuid.uuid4().hex
case_id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=case_id,
case_type='person',
version=V2,
).as_string()
submit_case_blocks(case_block, domain=self.domain, form_id=form_id)
# submit an edit form with a bad case update (for example a bad ID)
case_block = CaseBlock(
create=True,
case_id='',
case_type='person',
version=V2,
).as_string()
submit_case_blocks(case_block, domain=self.domain, form_id=form_id)
form = XFormInstance.get(form_id)
self.assertEqual('XFormError', form.doc_type)
deprecated_form = XFormInstance.get(form.deprecated_form_id)
self.assertEqual('XFormDeprecated', deprecated_form.doc_type)
def swap_doc_types(self, log_file, bad_xform_id, duplicate_xform_id, domain, dry_run):
bad_xform = XFormInstance.get(bad_xform_id)
# confirm that the doc hasn't already been fixed:
bad_xform_problem = None
try:
bad_xform_problem = bad_xform.problem or ""
except AttributeError:
pass
if bad_xform_problem:
if re.match(PROBLEM_TEMPLATE_START, bad_xform_problem):
self.log_already_fixed(log_file, bad_xform_id, domain)
return
duplicate_xform = XFormInstance.get(duplicate_xform_id)
now = datetime.now().isoformat()
# Convert the XFormInstance to an XFormDuplicate
bad_xform.doc_type = XFormDuplicate.__name__
bad_xform.problem = BAD_FORM_PROBLEM_TEMPLATE.format(duplicate_xform_id, now)
bad_xform = XFormDuplicate.wrap(bad_xform.to_json())
# Convert the XFormDuplicate to an XFormInstance
duplicate_xform.problem = FIXED_FORM_PROBLEM_TEMPLATE.format(
id_=bad_xform_id, datetime_=now
)
duplicate_xform.doc_type = XFormInstance.__name__
duplicate_xform = XFormInstance.wrap(duplicate_xform.to_json())
self.log_swap(log_file, bad_xform_id, domain, duplicate_xform_id, dry_run)
if not dry_run:
duplicate_xform.save()
bad_xform.save()
class StockReportDomainTest(TestCase):
def _get_name_for_domain(self):
return ''.join(
random.choice(string.ascii_lowercase)
for _ in range(DOMAIN_MAX_LENGTH)
)
def setUp(self):
self.domain = self._get_name_for_domain()
self.form = XFormInstance(domain=self.domain)
self.form.save()
self.new_stock_report = NewStockReport(
self.form,
datetime.now(),
REPORT_TYPE_BALANCE,
[],
)
def tearDown(self):
self.form.delete()
StockReport.objects.all().delete()
def test_stock_report(self):
self.new_stock_report.create_models()
filtered_stock_report = StockReport.objects.filter(domain=self.domain)
self.assertEquals(filtered_stock_report.count(), 1)
stock_report = filtered_stock_report.get()
self.assertEquals(stock_report.form_id, self.form._id)
self.assertEquals(stock_report.domain, self.domain)
def eval_dots_block(xform_json, callback=None):
"""
Evaluate the dots block in the xform submission and put it in the computed_ block for the xform.
"""
case_id = get_case_id(xform_json)
do_continue = False
#first, set the pact_data to json if the dots update stuff is there.
try:
if xform_json.get(PACT_DOTS_DATA_PROPERTY, {}).has_key('processed'):
#already processed, skipping
return
xform_json[PACT_DOTS_DATA_PROPERTY] = {}
if not isinstance(xform_json['form']['case'].get('update', None), dict):
#no case update property, skipping
pass
else:
#update is a dict
if xform_json['form']['case']['update'].has_key('dots'):
dots_json = xform_json['form']['case']['update']['dots']
if isinstance(dots_json, str) or isinstance(dots_json, unicode):
json_data = simplejson.loads(dots_json)
xform_json[PACT_DOTS_DATA_PROPERTY]['dots'] = json_data
do_continue=True
else:
#no dots data in doc
pass
xform_json[PACT_DOTS_DATA_PROPERTY]['processed']=datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
XFormInstance.get_db().save_doc(xform_json)
except Exception, ex:
#if this gets triggered, that's ok because web entry don't got them
tb = traceback.format_exc()
notify_exception(None, message="PACT error evaluating DOTS block docid %s, %s\n\tTraceback: %s" % (xform_json['_id'], ex, tb))
def _get_submissions_for_patient_by_date(patient, visit_dates, schema='http://dev.commcarehq.org/pact/dots_form'):
"""Argument: Patient django object, visit date
Will return a view result of all submissions by patient where the key is the patient pact_id
return value: [pact_id, year, month, day]=>submission"""
keys = []
date_key_map = {}
#t2 = datetime.now()
for visit_date in visit_dates:
day_of_week = visit_date.isoweekday()-1
yearstart = visit_date.year
monthstart = visit_date.month
datestart = visit_date.day
#get the xform count for that day
key = [patient.couchdoc.pact_id, yearstart, monthstart, datestart, schema]
keys.append(key)
key_str = ''.join([str(x) for x in key])
date_key_map[key_str] = visit_date
submit_reduction = XFormInstance.view('pactcarehq/all_submits_by_patient_date', keys=keys)
#d2 = datetime.now()-t2
#print "\tSingle Patient data query QUERY: %d.%d" % (d2.seconds, d2.microseconds/1000)
#t3 = datetime.now()
ret = {} #a return value of date ordered submissions by
for row in submit_reduction:
key = row['key']
key_str = ''.join([str(x) for x in key])
submits = row['value']
date = date_key_map[key_str]
ret[date] = [XFormInstance.wrap(x) for x in submits]
#d3 = datetime.now()-t3
#print "\tSingle Patient data query HASHING: %d.%d" % (d3.seconds, d3.microseconds/1000)
return ret
def test_form_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
form_id = self._save_doc_to_db('indicator_form.json', XFormInstance)
form_instance = XFormInstance.get(form_id)
# Form Label Indicator
form_label = FormLabelIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='create_form',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
form_label.save()
# Form Alias
form_alias = FormDataAliasIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
form_alias.save()
self.form_pillow.process_changes(since=since, forever=False)
indicator_form = IndicatorXForm.get(form_id)
self.assertNotEqual(
indicator_form.get_db().dbname, form_instance.get_db().dbname
)
self.assertNotEqual(indicator_form.computed_, {})
def test_assign_noop(self):
self._make_tree()
num_forms = XFormInstance.get_db().view('hqadmin/forms_over_time').all()[0]['value']
res = assign_case(self.primary, self.original_owner._id, include_subcases=True, include_parent_cases=True)
self.assertEqual(0, len(res))
new_num_forms = XFormInstance.get_db().view('hqadmin/forms_over_time').all()[0]['value']
self.assertEqual(new_num_forms, num_forms)
def setUpClass(cls):
super(TestCouchDomainFormChangeProvider, cls).setUpClass()
cls.domains = ['d1', 'd2', 'd3']
cls.form_ids = {
(domain, doc_type): ['{}-{}-{}'.format(doc_type, domain, i) for i in range(3)]
for domain in cls.domains
for doc_type in ['XFormInstance', 'XFormArchived']
}
cls.couch_db = FakeCouchDb(views={
"by_domain_doc_type_date/view":
[
(
{
'startkey': list(domain_doc_type),
'endkey': list(domain_doc_type) + [{}],
'include_docs': True,
'limit': 1000,
'reduce': False
},
[cls._get_row(form_id, domain_doc_type[0], domain_doc_type[1]) for form_id in form_ids]
)
for domain_doc_type, form_ids in cls.form_ids.items()
]
})
XFormInstance.set_db(cls.couch_db)
def tearDown(self):
try:
XFormInstance.get_db().delete_doc(self.ID)
except ResourceNotFound:
pass
deprecated_xforms = XFormDeprecated.view("couchforms/edits", include_docs=True).all()
for form in deprecated_xforms:
form.delete()
def setUp(self):
self.database = FakeCouchDb()
self.case_orig_db = CommCareCase.get_db()
self.form_orig_db = XFormInstance.get_db()
self.user_orig_db = CommCareUser.get_db()
CommCareCase.set_db(self.database)
XFormInstance.set_db(self.database)
CommCareUser.set_db(self.database)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
docs = list(processed_forms) + (cases or [])
docs = filter(None, docs)
assert XFormInstance.get_db().uri == CommCareCase.get_db().uri
with bulk_atomic_blobs(docs):
XFormInstance.get_db().bulk_save(docs)
if stock_result:
stock_result.commit()
def tearDown(self):
try:
XFormInstance.get_db().delete_doc(self.ID)
except ResourceNotFound:
pass
deprecated_xforms = access_edits(include_docs=True).all()
for form in deprecated_xforms:
form.delete()
def resave_form(domain, form):
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.change_publishers import publish_form_saved
from couchforms.models import XFormInstance
if should_use_sql_backend(domain):
publish_form_saved(form)
else:
XFormInstance.get_db().save_doc(form.to_json())
def forms_by_case(self):
assert self.cases is not None, "SharedDataProvider was not instantiated with cases"
all_form_ids = chain(*(case.xform_ids for case in self.cases))
forms_by_case = defaultdict(list)
for form in iter_docs(XFormInstance.get_db(), all_form_ids):
if form["xmlns"] in OPM_XMLNSs:
case_id = form["form"]["case"]["@case_id"]
forms_by_case[case_id].append(XFormInstance.wrap(form))
return forms_by_case
def is_duplicate(cls, xform_id, domain=None):
if domain:
try:
existing_doc = XFormInstance.get_db().get(xform_id)
except ResourceNotFound:
return False
return existing_doc.get('domain') == domain and existing_doc.get('doc_type') in doc_types()
else:
return xform_id in XFormInstance.get_db()
def sync_stock_transactions(domain, endpoint, facilities, checkpoint, date, limit=100, offset=0):
# todo: should figure out whether there's a better thing to be doing than faking this global form
try:
xform = XFormInstance.get(docid='ilsgateway-xform')
except ResourceNotFound:
xform = XFormInstance(_id='ilsgateway-xform')
xform.save()
for facility in facilities:
sync_stock_transactions_for_facility(domain, endpoint, facility, xform, checkpoint, date, limit, offset)
offset = 0 # reset offset for each facility, is only set in the context of a checkpoint resume
def form_exists(form_id, domain=None):
if not domain:
return XFormInstance.get_db().doc_exist(form_id)
else:
try:
xform = XFormInstance.get(form_id)
except ResourceNotFound:
return False
return xform.domain == domain
def reprocess_form_cases(form, config=None, case_db=None):
"""
For a given form, reprocess all case elements inside it. This operation
should be a no-op if the form was sucessfully processed, but should
correctly inject the update into the case history if the form was NOT
successfully processed.
"""
from casexml.apps.case.xform import process_cases, process_cases_with_casedb
if case_db:
process_cases_with_casedb([form], case_db, config=config)
else:
process_cases(form, config)
# mark cleaned up now that we've reprocessed it
if form.doc_type != 'XFormInstance':
form = XFormInstance.get(form._id)
form.doc_type = 'XFormInstance'
form.save()
def _copy_unprocessed_forms(self):
pool = Pool(10)
for couch_form_json in iter_docs(XFormInstance.get_db(), self.errors_with_normal_doc_type, chunksize=1000):
assert couch_form_json['problem']
couch_form_json['doc_type'] = 'XFormError'
pool.spawn(self._migrate_unprocessed_form, couch_form_json)
changes = _get_unprocessed_form_iterator(
self.domain).iter_all_changes(
resumable_key=self._get_resumable_iterator_key('unprocessed_forms'))
for change in self._with_progress(UNPROCESSED_DOC_TYPES, changes):
couch_form_json = change.get_document()
pool.spawn(self._migrate_unprocessed_form, couch_form_json)
while not pool.join(timeout=10):
self.log_info('Waiting on {} docs'.format(len(pool)))
self._log_unprocessed_forms_processed_count()
def test_edited_deleted_form(self):
form = create_and_save_a_form(self.domain_name)
form.edited_on = datetime.utcnow() - timedelta(days=400)
form.save()
FormAccessors(self.domain.name).soft_delete_forms([form.form_id],
datetime.utcnow(),
'test-deletion')
self.assertEqual(
get_doc_ids_in_domain_by_type(form.domain, "XFormInstance-Deleted",
XFormInstance.get_db()),
[form.form_id],
)
self._do_migration_and_assert_flags(form.domain)
self.assertEqual(
FormAccessorSQL.get_deleted_form_ids_in_domain(form.domain),
[form.form_id],
)
self._compare_diffs([])
def setUp(self):
self.domain_name = 'test'
self.password = "******"
username = "******"
email = "*****@*****.**"
self.user = WebUser.create(self.domain_name, username, self.password, email)
self.xforms = {}
for key, _id, in self.XFORMS.iteritems():
self.xforms[_id] = XFormInstance(xmlns='fake-xmlns',
domain=self.domain_name,
received_on=datetime.utcnow(),
form={
'#type': 'fake-type',
'@xmlns': 'fake-xmlns'
})
self.xforms[_id]['_id'] = _id
self.xforms[_id].save()
def _get_mvp_indicator_pillow(pillow_id, processor):
checkpoint = PillowCheckpoint(
'mvp_docs.pillows.{}.{}'.format(pillow_id, get_machine_id()), )
feed = CouchChangeFeed(XFormInstance.get_db(),
include_docs=True,
couch_filter='hqadmin/domains_and_doc_types',
extra_couch_view_params={
'domains': ' '.join(processor.domains),
'doc_types': ' '.join(processor.doc_types),
})
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=feed,
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100),
)
def setUp(self):
for item in get_all_forms_in_all_domains():
item.delete()
for item in FormData.objects.all():
item.delete()
file_path = os.path.join(os.path.dirname(__file__), "data", "meta.xml")
with open(file_path, "rb") as f:
xml_data = f.read()
SubmissionPost(instance=xml_data,
domain='sofabed',
app_id='12345',
received_on=datetime.utcnow()).get_response()
self.instance = XFormInstance.get('THIS_IS_THE_INSTANCEID')
def process_xform(self, doc):
"""Process XFormInstance document asynchronously"""
form_id = doc["_id"]
log.debug('Processing doc: XFormInstance(%s)', form_id)
if doc.get('problem'):
if str(doc['problem']).startswith(PROBLEM_TEMPLATE_START):
doc = _fix_replacement_form_problem_in_couch(doc)
else:
self.statedb.add_problem_form(form_id)
return
try:
wrapped_form = XFormInstance.wrap(doc)
except Exception:
log.exception("Error migrating form %s", form_id)
self.statedb.save_form_diffs(doc, {})
else:
self._try_to_process_form(wrapped_form)
self._try_to_empty_queues()
def populate_dup_map(self, domain):
self.dups_by_domain[domain] = defaultdict(set)
dups = XFormInstance.view(
'couchforms/all_submissions_by_domain',
startkey=[domain, 'XFormDuplicate', '2016'],
endkey=[domain, "XFormDuplicate", {}],
reduce=False,
include_docs=True
)
for dup in dups:
match = re.match(r'Form is a duplicate of another! \((.*)\)', dup.problem or "")
if match:
orig_id = match.groups()[0]
try:
orig_id = UUID(orig_id)
except ValueError:
continue
self.dups_by_domain[domain][orig_id].add(dup._id)
def get_forms_by_type(domain, type_, recent_first=False, limit=None):
assert type_ in doc_types()
# no production code should be pulling all forms in one go!
assert limit is not None
startkey = [domain, type_]
endkey = startkey + [{}]
if recent_first:
startkey, endkey = endkey, startkey
return XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
descending=recent_first,
include_docs=True,
limit=limit,
classes=doc_types(),
).all()
def test_update_dependent_case(self):
sync_log = SimplifiedSyncLog(
case_ids_on_phone={'bran', 'hodor'},
dependent_case_ids_on_phone={'hodor'},
index_tree=IndexTree(indices={'bran': {
'legs': 'hodor'
}}),
user_id="someuser")
xform_id = uuid.uuid4().hex
xform = XFormInstance(_id=xform_id)
form_actions = [CommCareCaseAction(action_type=CASE_ACTION_UPDATE, )]
with patch.object(CommCareCase,
'get_actions_for_form',
return_value=form_actions):
parent_case = CommCareCase(_id='hodor')
# before this test was added, the following call raised a SyncLogAssertionError on legacy logs.
# this test just ensures it doesn't still do that.
sync_log.update_phone_lists(xform, [parent_case])
def _submitAndVerifyBundle(self, bundle, verify=True):
start_nums = len(self.case.xform_ids)
submit_xform(self.submit_url, self.domain.name, bundle['xml'])
time.sleep(1)
submitted = XFormInstance.get(bundle['xform_id'])
self.assertTrue(hasattr(submitted, PACT_DOTS_DATA_PROPERTY))
submitted_dots = getattr(submitted, PACT_DOTS_DATA_PROPERTY)
updated_case = PactPatientCase.get(CASE_ID)
case_dots = get_dots_case_json(updated_case)
days = case_dots['days']
if verify:
nonart_submissions = bundle['nonart']
art_submissions = bundle['art']
examine_day = days[bundle['check_idx']]
self._verify_dot_cells(nonart_submissions, art_submissions, examine_day)
def post_xform_to_couch(instance, attachments={}):
"""
Post an xform to couchdb, based on the settings.XFORMS_POST_URL.
Returns the newly created document from couchdb, or raises an
exception if anything goes wrong.
attachments is a dictionary of the request.FILES that are not the xform. Key is the parameter name, and the value is the django MemoryFile object stream.
"""
def _has_errors(response, errors):
return errors or "error" in response
# check settings for authentication credentials
try:
response, errors = post_from_settings(instance)
if not _has_errors(response, errors):
doc_id = response
try:
xform = XFormInstance.get(doc_id)
#put attachments onto the saved xform instance
for key, val in attachments.items():
res = xform.put_attachment(val,
name=key,
content_type=val.content_type,
content_length=val.size)
# fire signals
# We don't trap any exceptions here. This is by design.
# If something fails (e.g. case processing), we quarantine the
# form into an error location.
xform_saved.send(sender="couchforms", xform=xform)
return xform
except Exception, e:
logging.error("Problem with form %s" % doc_id)
logging.exception(e)
# "rollback" by changing the doc_type to XFormError
try:
bad = XFormError.get(doc_id)
bad.problem = "%s" % e
bad.save()
return bad
except ResourceNotFound:
pass # no biggie, the failure must have been in getting it back
raise
else:
def _process_main_forms(self):
last_received_on = datetime.min
# form_id needs to be on self to release appropriately
self.queues = PartiallyLockingQueue("form_id", max_size=10000, run_timestamp=self.run_timestamp)
pool = Pool(15)
self._rebuild_queues(pool)
# process main forms (including cases and ledgers)
changes = _get_main_form_iterator(
self.domain).iter_all_changes(
resumable_key=self._get_resumable_iterator_key('main_forms'))
# form_id needs to be on self to release appropriately
self.queues = PartiallyLockingQueue("form_id", max_size=10000)
for change in self._with_progress(['XFormInstance'], changes):
self.log_debug('Processing doc: {}({})'.format('XFormInstance', change.id))
form = change.get_document()
if form.get('problem', None):
self.errors_with_normal_doc_type.append(change.id)
continue
wrapped_form = XFormInstance.wrap(form)
form_received = wrapped_form.received_on
assert last_received_on <= form_received
last_received_on = form_received
self._try_to_process_form(wrapped_form, pool)
self._try_to_process_queues(pool)
# finish up the queues once all changes have been iterated through
while self.queues.has_next():
wrapped_form = self.queues.get_next()
if wrapped_form:
pool.spawn(self._migrate_form_and_associated_models_async, wrapped_form)
else:
gevent.sleep(0.01) # swap greenlets
remaining_items = self.queues.remaining_items + len(pool)
if remaining_items % 10 == 0:
self.log_info('Waiting on {} docs'.format(remaining_items))
while not pool.join(timeout=10):
self.log_info('Waiting on {} docs'.format(len(pool)))
self._log_main_forms_processed_count()
def change_submissions_app_id(request, domain):
app_id = request.POST['app_id'] or None
xmlns = request.POST['xmlns']
new_app_id = request.POST['new_app_id']
next = request.POST['next']
submissions = XFormInstance.view('exports_forms/by_xmlns',
key=['^XFormInstance', domain, app_id, xmlns],
include_docs=True,
reduce=False,
).all()
for sub in submissions:
assert(getattr(sub, 'app_id', None) == app_id)
assert(sub.xmlns == xmlns)
sub.app_id = new_app_id
sub.save()
messages.success(request, 'Your fix was successful and affected %s submissions' % len(submissions))
return HttpResponseRedirect(next)
def get_form_ids_by_type(domain, type_, start=None, end=None):
assert type_ in doc_types()
startkey = [domain, type_]
if end:
endkey = startkey + [end.isoformat()]
else:
endkey = startkey + [{}]
if start:
startkey.append(start.isoformat())
return [
row['id'] for row in XFormInstance.get_db().view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
)
]
def test_submit_bad_case_id(self):
instance_id = uuid.uuid4().hex
amounts = [(p._id, float(i*10)) for i, p in enumerate(self.products)]
xml_stub = balance_submission(amounts)
instance = submission_wrap(
instance_id,
self.products,
self.user,
'missing',
'missing-too',
xml_stub,
)
submit_form_locally(
instance=instance,
domain=self.domain.name,
)
form = XFormInstance.get(instance_id)
self.assertEqual('XFormError', form.doc_type)
self.assertTrue('IllegalCaseId' in form.problem)
def _process_main_forms(self):
last_received_on = datetime.min
# process main forms (including cases and ledgers)
changes = _get_main_form_iterator(self.domain).iter_all_changes()
for change in self._with_progress(['XFormInstance'], changes):
self.log_debug('Processing doc: {}({})'.format('XFormInstance', change.id))
form = change.get_document()
if form.get('problem', None):
self.errors_with_normal_doc_type.append(change.id)
continue
wrapped_form = XFormInstance.wrap(form)
form_received = wrapped_form.received_on
assert last_received_on <= form_received
last_received_on = form_received
try:
self._migrate_form_and_associated_models(wrapped_form)
except:
self.log_error("Unable to migrate form: {}".format(change.id))
raise
def _with_progress(self, doc_types, iterable, progress_name='Migrating'):
doc_count = sum([
get_doc_count_in_domain_by_type(self.domain, doc_type,
XFormInstance.get_db())
for doc_type in doc_types
])
if self.timing_context:
current_timer = self.timing_context.peek()
current_timer.normalize_denominator = doc_count
if self.with_progress:
prefix = "{} ({})".format(progress_name, ', '.join(doc_types))
return with_progress_bar(iterable,
doc_count,
prefix=prefix,
oneline=False)
else:
log.info("{} ({})".format(doc_count, ', '.join(doc_types)))
return iterable
def test_deleted_form_migration(self):
form = create_and_save_a_form(self.domain_name)
FormAccessors(self.domain.name).soft_delete_forms([form.form_id],
datetime.utcnow(),
'test-deletion')
self.assertEqual(
1,
len(
get_doc_ids_in_domain_by_type(self.domain_name,
"XFormInstance-Deleted",
XFormInstance.get_db())))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(
1,
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain_name)))
self._compare_diffs([])
def forms_for_cases_for_users(user_ids, domain):
all_form_ids = set()
for user_id in user_ids:
user = CommCareUser.get_by_user_id(user_id, domain=domain)
all_form_ids.update(user.get_forms(wrap=False))
for case in user.get_cases(last_submitter=True):
all_form_ids.update(case.xform_ids)
forms = []
for id in all_form_ids:
try:
form = XFormInstance.get(id)
except ResourceNotFound:
continue
else:
if check_form_domain(form, domain):
forms.append(form)
forms.sort(key=lambda form: form.received_on)
return forms
def iter_unmigrated_docs(domain, doc_types, migration_id, counter):
if doc_types != ["XFormInstance"]:
raise NotImplementedError(doc_types)
[doc_type] = doc_types
couch_db = XFormInstance.get_db()
doc_count = counter.pop(doc_type)
if doc_count:
log.info("saved count of %s was %s", doc_type, doc_count)
doc_count = get_doc_count_in_domain_by_type(domain, doc_type, couch_db)
add_docs = partial(counter.add, None, doc_type)
batches = doc_count // iter_id_chunks.chunk_size
iterable = iter_id_chunks(domain, doc_type, migration_id, couch_db)
doc_ids = []
for doc_ids in with_progress_bar(iterable,
batches,
prefix=doc_type,
oneline=False):
yield from iter_docs_not_in_sql(doc_ids, couch_db)
add_docs(len(doc_ids))
def get_data(self):
# todo: this will probably have to paginate eventually
if self.all_relevant_forms:
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location,
)
supply_points = (SupplyPointCase.wrap(doc) for doc in iter_docs(SupplyPointCase.get_db(), sp_ids))
form_xmlnses = [form['xmlns'] for form in self.all_relevant_forms.values()]
for supply_point in supply_points:
# todo: get locations in bulk
loc = supply_point.location
transactions = StockTransaction.objects.filter(
case_id=supply_point._id,
).exclude(
report__date__lte=self.start_date
).exclude(
report__date__gte=self.end_date
).order_by('-report__date')
matched = False
for trans in transactions:
if XFormInstance.get(trans.report.form_id).xmlns in form_xmlnses:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': 'reporting',
'geo': loc._geopoint,
}
matched = True
break
if not matched:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': 'nonreporting',
'geo': loc._geopoint,
}
def _add_eligible_9months(row_data, start_date, end_date, domain):
eligible_9months = McctStatus.objects\
.filter(domain__exact=domain)\
.filter(status__exact="eligible")\
.filter(immunized=False)\
.filter(is_booking=False)\
.filter(received_on__range=(start_date, end_date))
forms = [
form for form in
[XFormInstance.get(status.form_id) for status in eligible_9months]
if form.xmlns in FOLLOW_UP_FORMS
]
forms_4th_visit = [
form for form in forms if form.form.get("visits", "") == "4"
]
row_data["all_eligible_clients_total"]["value"] += len(forms) - len(
forms_4th_visit)
row_data["status_eligible_due_to_4th_visit"]["value"] += len(
forms_4th_visit)
def test_update_dependent_case(self):
sync_log = SyncLog(
cases_on_phone=[
CaseState(
case_id='bran',
indices=[CommCareCaseIndex(identifier='legs', referenced_id='hodor')],
),
],
dependent_cases_on_phone=[CaseState(case_id='hodor')]
)
xform_id = uuid.uuid4().hex
xform = XFormInstance(_id=xform_id)
form_actions = [CommCareCaseAction(action_type=CASE_ACTION_UPDATE,)]
with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions):
parent_case = CommCareCase(_id='hodor')
# before this test was added, the following call raised a SyncLogAssertionError on legacy logs.
# this test just ensures it doesn't still do that.
for log in [sync_log, SimplifiedSyncLog.from_other_format(sync_log)]:
log.update_phone_lists(xform, [parent_case])
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationField.slug, '')
UNKNOWN = _("unknown")
for user in self.users:
last_seen = self.table_cell(-1, _("Never"))
app_name = "---"
is_unknown = True
key = make_form_couch_key(self.domain, user_id=user.get('user_id'))
data = XFormInstance.view("reports_forms/all_forms",
startkey=key+[{}],
endkey=key,
include_docs=True,
descending=True,
reduce=False,
limit=1,
).first()
if data:
last_seen = util.format_relative_date(data.received_on)
if data.version != '1':
build_id = data.version
else:
build_id = UNKNOWN
form_data = data.get_form
try:
app_name = form_data['meta']['appVersion']['#text']
except KeyError:
try:
app = Application.get(data.app_id)
is_unknown = False
if selected_app and selected_app != data.app_id:
continue
app_name = "%s [%s]" % (app.name, build_id)
except Exception:
app_name = UNKNOWN
if is_unknown and selected_app:
continue
row = [user.get('username_in_report'), last_seen, app_name]
rows.append(row)
return rows
def new_form_from_old(cls, existing_form, xml, value_responses_map,
user_id):
from corehq.form_processor.parsers.form import apply_deprecation
new_form = XFormInstance.wrap(existing_form.to_json())
for question, response in six.iteritems(value_responses_map):
data = new_form.form_data
i = XFormQuestionValueIterator(question)
for (qid, repeat_index) in i:
data = data[qid]
if repeat_index is not None:
data = data[repeat_index]
data[i.last()] = response
new_form._deferred_blobs = None # will be re-populated by apply_deprecation
new_form.external_blobs.clear(
) # will be re-populated by apply_deprecation
existing_form, new_form = apply_deprecation(existing_form, new_form)
return (existing_form, new_form)
def update_indicators_for_xmlns(self, domain, form_label_filter=None):
key = [MVP.NAMESPACE, domain]
all_labels = get_db().view(
'indicators/form_labels',
reduce=False,
startkey=key,
endkey=key + [{}],
).all()
for label in all_labels:
label_name = label['value']
if form_label_filter is not None and form_label_filter != label_name:
continue
xmlns = label['key'][-2]
print "\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (
label_name, xmlns, domain)
relevant_forms = get_db().view(
"reports_forms/all_forms",
reduce=True,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).first()
num_forms = relevant_forms['value'] if relevant_forms else 0
form_ids = [
r['id'] for r in XFormInstance.view(
"reports_forms/all_forms",
reduce=False,
include_docs=False,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).all()
]
print "Found %d forms with matching XMLNS %s" % (num_forms, xmlns)
relevant_indicators = FormIndicatorDefinition.get_all(
namespace=MVP.NAMESPACE, domain=domain, xmlns=xmlns)
if relevant_indicators:
self._throttle_updates(
"Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" %
(label_name, xmlns, domain), relevant_indicators,
num_forms, domain, form_ids, XFormInstance)
def _test(self, form, response_contains, xmlns, msg=None):
response = self._submit(form,
HTTP_DATE='Mon, 11 Apr 2011 18:24:43 GMT')
xform_id = response['X-CommCareHQ-FormID']
foo = XFormInstance.get(xform_id).to_json()
n_times_saved = int(foo['_rev'].split('-')[0])
self.assertTrue(foo['received_on'])
for key in ['form', '_attachments', '_rev', 'received_on']:
del foo[key]
self.assertEqual(n_times_saved, 1)
self.assertEqual(foo, {
"#export_tag": [
"domain",
"xmlns"
],
"_id": xform_id,
"app_id": None,
"auth_context": {
"authenticated": False,
"doc_type": "AuthContext",
"domain": "submit",
"user_id": None
},
"build_id": None,
"computed_": {},
"computed_modified_on_": None,
"date_header": '2011-04-11T18:24:43.000000Z',
"doc_type": "XFormInstance",
"domain": "submit",
"history": [],
"initial_processing_complete": True,
"last_sync_token": None,
"openrosa_headers": {
"HTTP_DATE": "Mon, 11 Apr 2011 18:24:43 GMT",
},
"partial_submission": False,
"path": "/a/submit/receiver",
"submit_ip": "127.0.0.1",
"xmlns": xmlns,
})
self.assertIn(response_contains, str(response), msg)
def _iter_docs(domain, doc_type, resume_key, stopper):
@retry_on_couch_error
def data_function(**view_kwargs):
view_name = 'by_domain_doc_type_date/view'
results = list(couch_db.view(view_name, **view_kwargs))
assert all(r['key'][0] == domain for r in results), \
_repr_bad_results(view_name, view_kwargs, results, domain)
return results
if "." in doc_type:
doc_type, row_key = doc_type.split(".")
else:
row_key = "doc"
if stopper.clean_break:
return []
couch_db = XFormInstance.get_db()
args_provider = NoSkipArgsProvider({
'startkey': [domain, doc_type],
'endkey': [domain, doc_type, {}],
'limit': _iter_docs.chunk_size,
'include_docs': row_key == "doc",
'reduce': False,
})
rows = ResumableFunctionIterator(
resume_key,
data_function,
args_provider,
item_getter=None,
event_handler=MigrationPaginationEventHandler(domain, stopper))
if rows.state.is_resume():
log.info("iteration state: %r", rows.state.to_json())
row = None
try:
for row in rows:
yield row[row_key]
finally:
if row is not None:
row_copy = dict(row)
row_copy.pop("doc", None)
log.info("last item: %r", row_copy)
log.info("final iteration state: %r", rows.state.to_json())
def apply_attachments(self, attachment_action):
# the actions and _attachment must be added before the first saves can happen
# todo attach cached attachment info
stream_dict = {}
# cache all attachment streams from xform
for k, v in attachment_action.attachments.items():
if v.is_present:
# fetch attachment, update metadata, get the stream
attach_data = XFormInstance.get_db().fetch_attachment(
attachment_action.xform_id, v.attachment_src)
stream_dict[k] = attach_data
v.attachment_size = len(attach_data)
if v.is_image:
img = Image.open(StringIO(attach_data))
img_size = img.size
props = dict(width=img_size[0], height=img_size[1])
v.attachment_properties = props
self.force_save()
update_attachments = {}
for k, v in self.case_attachments.items():
if v.is_present:
update_attachments[k] = v
for k, v in attachment_action.attachments.items():
#grab xform_attachments
#copy over attachments from form onto case
update_attachments[k] = v
if v.is_present:
#fetch attachment from xform
attachment_key = v.attachment_key
attach = stream_dict[attachment_key]
self.put_attachment(attach,
name=attachment_key,
content_type=v.server_mime)
else:
self.delete_attachment(k)
del (update_attachments[k])
self.case_attachments = update_attachments
