def eval_dots_block(xform_json, callback=None):
"""
Evaluate the dots block in the xform submission and put it in the computed_ block for the xform.
"""
case_id = get_case_id(xform_json)
do_continue = False
#first, set the pact_data to json if the dots update stuff is there.
try:
if xform_json.get(PACT_DOTS_DATA_PROPERTY, {}).has_key('processed'):
#already processed, skipping
return
xform_json[PACT_DOTS_DATA_PROPERTY] = {}
if not isinstance(xform_json['form']['case'].get('update', None), dict):
#no case update property, skipping
pass
else:
#update is a dict
if xform_json['form']['case']['update'].has_key('dots'):
dots_json = xform_json['form']['case']['update']['dots']
if isinstance(dots_json, str) or isinstance(dots_json, unicode):
json_data = simplejson.loads(dots_json)
xform_json[PACT_DOTS_DATA_PROPERTY]['dots'] = json_data
do_continue=True
else:
#no dots data in doc
pass
xform_json[PACT_DOTS_DATA_PROPERTY]['processed']=datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
XFormInstance.get_db().save_doc(xform_json)
except Exception, ex:
#if this gets triggered, that's ok because web entry don't got them
tb = traceback.format_exc()
notify_exception(None, message="PACT error evaluating DOTS block docid %s, %s\n\tTraceback: %s" % (xform_json['_id'], ex, tb))
def save_processed_models(cls, processed_forms, cases=None, stock_updates=None):
docs = list(processed_forms) + (cases or [])
docs = filter(None, docs)
assert XFormInstance.get_db().uri == CommCareCase.get_db().uri
XFormInstance.get_db().bulk_save(docs)
for stock_update in stock_updates or []:
stock_update.commit()
def test_assign_noop(self):
self._make_tree()
num_forms = XFormInstance.get_db().view('hqadmin/forms_over_time').all()[0]['value']
res = assign_case(self.primary, self.original_owner._id, include_subcases=True, include_parent_cases=True)
self.assertEqual(0, len(res))
new_num_forms = XFormInstance.get_db().view('hqadmin/forms_over_time').all()[0]['value']
self.assertEqual(new_num_forms, num_forms)
def tag_forms_as_deleted_rebuild_associated_cases(user_id, domain, form_id_list, deletion_id,
deletion_date, deleted_cases=None):
"""
Upon user deletion, mark associated forms as deleted and prep cases
for a rebuild.
- 2 saves/sec for cloudant slowness (rate_limit)
"""
if deleted_cases is None:
deleted_cases = set()
cases_to_rebuild = set()
forms_to_check = get_docs(XFormInstance.get_db(), form_id_list)
forms_to_save = []
for form in forms_to_check:
assert form['domain'] == domain
if not is_deleted(form):
form['doc_type'] += DELETED_SUFFIX
form['-deletion_id'] = deletion_id
form['-deletion_date'] = deletion_date
forms_to_save.append(form)
# rebuild all cases anyways since we don't know if this has run or not if the task was killed
cases_to_rebuild.update(get_case_ids_from_form(form))
XFormInstance.get_db().bulk_save(forms_to_save)
detail = UserArchivedRebuild(user_id=user_id)
for case in cases_to_rebuild - deleted_cases:
_rebuild_case_with_retries.delay(domain, case, detail)
def update_analytics_indexes():
"""
Mostly for testing; wait until analytics data sources are up to date
so that calls to analytics functions return up-to-date
"""
XFormInstance.get_db().view("reports_forms/all_forms", limit=1).all()
XFormInstance.get_db().view("exports_forms/by_xmlns", limit=1).all()
def test_basic_edit(self):
xml_data1, xml_data2 = self._get_files()
docs = []
doc = post_xform_to_couch(xml_data1)
self.assertEqual(self.ID, doc.get_id)
self.assertEqual("XFormInstance", doc.doc_type)
self.assertEqual("", doc.form["vitals"]["height"])
self.assertEqual("other", doc.form["assessment"]["categories"])
doc.domain = "test-domain"
doc.save()
doc = post_xform_to_couch(xml_data2, domain="test-domain")
self.assertEqual(self.ID, doc.get_id)
self.assertEqual("XFormInstance", doc.doc_type)
self.assertEqual("100", doc.form["vitals"]["height"])
self.assertEqual("Edited Baby!", doc.form["assessment"]["categories"])
docs.append(doc)
doc = XFormDeprecated.view("couchforms/edits", include_docs=True).first()
self.assertEqual(self.ID, doc.orig_id)
self.assertNotEqual(self.ID, doc.get_id)
self.assertEqual(XFormDeprecated.__name__, doc.doc_type)
self.assertEqual("", doc.form["vitals"]["height"])
self.assertEqual("other", doc.form["assessment"]["categories"])
self.assertEqual(XFormInstance.get_db().fetch_attachment(doc.get_id, "form.xml"), xml_data1)
self.assertEqual(XFormInstance.get_db().fetch_attachment(self.ID, "form.xml"), xml_data2)
for doc in docs:
doc.delete()
def process_cases(xform, config=None):
"""
Creates or updates case objects which live outside of the form.
If reconcile is true it will perform an additional step of
reconciling the case update history after the case is processed.
"""
warnings.warn(
'This function is deprecated. You should be using SubmissionPost.',
DeprecationWarning,
)
assert getattr(settings, 'UNIT_TESTING', False)
domain = get_and_check_xform_domain(xform)
with CaseDbCache(domain=domain, lock=True, deleted_ok=True) as case_db:
case_result = process_cases_with_casedb([xform], case_db, config=config)
cases = case_result.cases
docs = [xform] + cases
now = datetime.datetime.utcnow()
for case in cases:
case.server_modified_on = now
XFormInstance.get_db().bulk_save(docs)
for case in cases:
case_post_save.send(CommCareCase, case=case)
case_result.commit_dirtiness_flags()
return cases
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
docs = list(processed_forms) + (cases or [])
docs = filter(None, docs)
assert XFormInstance.get_db().uri == CommCareCase.get_db().uri
with bulk_atomic_blobs(docs):
XFormInstance.get_db().bulk_save(docs)
if stock_result:
stock_result.commit()
def resave_form(domain, form):
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.change_publishers import publish_form_saved
from couchforms.models import XFormInstance
if should_use_sql_backend(domain):
publish_form_saved(form)
else:
XFormInstance.get_db().save_doc(form.to_json())
def tearDown(self):
try:
XFormInstance.get_db().delete_doc(self.ID)
except ResourceNotFound:
pass
deprecated_xforms = XFormDeprecated.view("couchforms/edits", include_docs=True).all()
for form in deprecated_xforms:
form.delete()
def tearDown(self):
try:
XFormInstance.get_db().delete_doc(self.ID)
except ResourceNotFound:
pass
deprecated_xforms = access_edits(include_docs=True).all()
for form in deprecated_xforms:
form.delete()
def is_duplicate(cls, xform_id, domain=None):
if domain:
try:
existing_doc = XFormInstance.get_db().get(xform_id)
except ResourceNotFound:
return False
return existing_doc.get('domain') == domain and existing_doc.get('doc_type') in doc_types()
else:
return xform_id in XFormInstance.get_db()
def test_wrong_domain(self):
domain = 'test-domain'
XFormInstance.get_db().save_doc({
'_id': self.ID,
'doc_type': 'XFormInstance',
'domain': 'wrong-domain',
})
doc = post_xform_to_couch(instance=self._get_file(), domain=domain)
self.assertNotEqual(doc.get_id, self.ID)
def test_broken_save(self):
"""
Test that if the second form submission terminates unexpectedly
and the main form isn't saved, then there are no side effects
such as the original having been marked as deprecated.
"""
class BorkDB(object):
"""context manager for making a db's bulk_save temporarily fail"""
def __init__(self, db):
self.old = {}
self.db = db
def __enter__(self):
self.old['bulk_save'] = self.db.bulk_save
self.db.bulk_save = MagicMock(name='bulk_save',
side_effect=RequestFailed())
def __exit__(self, exc_type, exc_val, exc_tb):
self.db.bulk_save = self.old['bulk_save']
self.assertEqual(
XFormInstance.view('couchforms/edits', key=self.ID).count(), 0)
self.assertFalse(XFormInstance.get_db().doc_exist(self.ID))
xml_data1, xml_data2 = self._get_files()
submit_form_locally(xml_data1, self.domain)
doc = XFormInstance.get(self.ID)
self.assertEqual(self.ID, doc.get_id)
self.assertEqual("XFormInstance", doc.doc_type)
self.assertEqual(self.domain, doc.domain)
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
0
)
with BorkDB(XFormInstance.get_db()):
with self.assertRaises(RequestFailed):
submit_form_locally(xml_data2, self.domain)
# it didn't go through, so make sure there are no edits still
self.assertEqual(
XFormInstance.view('couchforms/edits', key=self.ID).count(), 0)
self.assertTrue(XFormInstance.get_db().doc_exist(self.ID))
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID,
saved=False).count(),
1
)
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
1
)
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def test_wrong_doc_type(self):
domain = 'test-domain'
id = '7H46J37FGH3'
XFormInstance.get_db().save_doc({
'_id': id,
'doc_type': 'Foo',
'domain': domain,
})
doc = post_xform_to_couch(instance=self._get_file(), domain=domain)
self.assertNotEqual(doc.get_id, id)
def test_wrong_doc_type(self):
domain = 'test-domain'
XFormInstance.get_db().save_doc({
'_id': self.ID,
'doc_type': 'Foo',
'domain': domain,
})
self.addCleanup(lambda: XFormInstance.get_db().delete_doc(self.ID))
instance = self._get_file()
xform = FormProcessorInterface.post_xform(instance, domain=domain)
self.assertNotEqual(xform.id, self.ID)
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {location_id: name for (location_id, name) in locations}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def test_simple_delete(self):
factory = CaseFactory()
case = factory.create_case()
[case] = factory.create_or_update_case(CaseStructure(case_id=case._id, attrs={'update': {'foo': 'bar'}}))
self.assertTrue(CommCareCase.get_db().doc_exist(case._id))
self.assertEqual(2, len(case.xform_ids))
for form_id in case.xform_ids:
self.assertTrue(XFormInstance.get_db().doc_exist(form_id))
safe_hard_delete(case)
self.assertFalse(CommCareCase.get_db().doc_exist(case._id))
for form_id in case.xform_ids:
self.assertFalse(XFormInstance.get_db().doc_exist(form_id))
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
docs = list(processed_forms)
for form in docs:
if form:
form.server_modified_on = datetime.datetime.utcnow()
docs += (cases or [])
docs = [_f for _f in docs if _f]
assert XFormInstance.get_db().uri == CommCareCase.get_db().uri
with bulk_atomic_blobs(docs):
XFormInstance.get_db().bulk_save(docs)
if stock_result:
stock_result.commit()
def tag_forms_as_deleted_rebuild_associated_cases(formlist, deletion_id, deleted_cases=None):
from casexml.apps.case.cleanup import rebuild_case
if deleted_cases is None:
deleted_cases = set()
cases_to_rebuild = set()
for form in formlist:
form['doc_type'] += DELETED_SUFFIX
form['-deletion_id'] = deletion_id
cases_to_rebuild.update(get_case_ids_from_form(form))
XFormInstance.get_db().bulk_save(formlist)
for case in cases_to_rebuild - deleted_cases:
rebuild_case(case)
def form_multimedia_export(request, domain, app_id):
try:
xmlns = request.GET["xmlns"]
startdate = request.GET["startdate"]
enddate = request.GET["enddate"]
zip_name = request.GET.get("name", None)
except KeyError:
return HttpResponseBadRequest()
def filename(form, question_id, extension):
return "%s-%s-%s-%s.%s" % (form['form']['@name'],
unidecode(question_id),
form['form']['meta']['username'],
form['_id'], extension)
key = [domain, app_id, xmlns]
stream_file = cStringIO.StringIO()
zf = zipfile.ZipFile(stream_file, mode='w', compression=zipfile.ZIP_STORED)
size = 22 # overhead for a zipfile
unknown_number = 0
form_ids = {f['id'] for f in XFormInstance.get_db().view("attachments/attachments",
start_key=key + [startdate],
end_key=key + [enddate, {}],
reduce=False)}
for form in iter_docs(XFormInstance.get_db(), form_ids):
f = XFormInstance.wrap(form)
if not zip_name:
zip_name = unidecode(form['form']['@name'])
for key in form['_attachments'].keys():
if form['_attachments'][key]['content_type'] == 'text/xml':
continue
extension = unicode(os.path.splitext(key)[1])
try:
question_id = unicode('-'.join(find_question_id(form['form'], key)))
except TypeError:
question_id = unicode('unknown' + str(unknown_number))
unknown_number += 1
fname = filename(form, question_id, extension)
zi = zipfile.ZipInfo(fname, parse(form['received_on']).timetuple())
zf.writestr(zi, f.fetch_attachment(key, stream=True).read())
# includes overhead for file in zipfile
size += f['_attachments'][key]['length'] + 88 + 2 * len(fname)
zf.close()
response = HttpResponse(stream_file.getvalue(), mimetype="application/zip")
response['Content-Length'] = size
response['Content-Disposition'] = 'attachment; filename=%s.zip' % zip_name
return response
def historical_forms(request, domain):
assert request.couch_user.is_member_of(domain)
user_id = request.couch_user.get_id
db = XFormInstance.get_db()
form_ids = {
f['id'] for f in db.view(
'reports_forms/all_forms',
startkey=["submission user", domain, user_id],
endkey=["submission user", domain, user_id, {}],
reduce=False,
)
}
def data():
yield (
'<OpenRosaResponse xmlns="http://openrosa.org/http/response" '
'items="{}">\n <message nature="success"/>\n'
.format(len(form_ids))
)
for form_id in form_ids:
# this is a hack to call this method
# Should only hit couch once per form, to get the attachment
xml = XFormInstance(_id=form_id).get_xml_element()
if xml:
yield ' {}'.format(etree.tostring(xml))
else:
yield ' <XFormNotFound/>'
yield '\n'
yield '</OpenRosaResponse>\n'
# to make this not stream, just call list on data()
return HttpResponse(data(), content_type='application/xml')
def handle(self, *args, **options):
if len(args) < 2:
print "please specify a filepath and an archiving_user"
return
filepath = args[0]
archiving_user = args[1]
try:
form_ids = open(filepath)
except Exception as e:
print "there was an issue opening the file: %s" % e
return
try:
form_ids = [f[0] for f in csv.reader(form_ids)]
except Exception as e:
print "there was an issue reading the file %s" % e
return
for xform_doc in iter_docs(XFormInstance.get_db(), form_ids):
try:
xform = XFormInstance.wrap(xform_doc)
xform.archive(user_id=archiving_user)
print "Archived form %s in domain %s" % (
xform._id, xform.domain
)
except Exception as e:
print "Issue archiving XFORM %s for domain %s: %s" % (
xform_doc['_id'], xform_doc['domain'], e
)
def get_value(self, user_ids, datespan=None, is_debug=False):
if datespan:
enddate = datespan.enddate_utc
else:
enddate = datetime.datetime.utcnow()
days = []
for user_id in user_ids:
key = make_form_couch_key(self.domain, user_id=user_id)
results = XFormInstance.get_db().view("reports_forms/all_forms",
reduce=False,
include_docs=False,
descending=True,
startkey=key+[enddate.isoformat(), {}],
endkey=key,
limit=1
).first()
try:
last_transmission = results['key'][-1]
last_date = dateutil.parser.parse(last_transmission)
last_date = last_date.replace(tzinfo=pytz.utc)
enddate = enddate.replace(tzinfo=pytz.utc)
td = enddate - last_date
days.append(td.days)
except Exception:
pass
if len(days) == 1:
return days[0]
if not days:
return None
return days
def testUpdateWithNoNewAttachment(self):
_, case = self._doCreateCaseWithMultimedia()
bulk_save = XFormInstance.get_db().bulk_save
bulk_save_attachments = []
# pull out and record attachments to docs being bulk saved
def new_bulk_save(docs, *args, **kwargs):
for doc in docs:
if doc['_id'] == TEST_CASE_ID:
bulk_save_attachments.append(doc['_attachments'])
bulk_save(docs, *args, **kwargs)
self._doSubmitUpdateWithMultimedia(
new_attachments=[], removes=[])
with patch('couchforms.models.XFormInstance._db.bulk_save', new_bulk_save):
# submit from the 2 min in the past to trigger a rebuild
self._doSubmitUpdateWithMultimedia(
new_attachments=[], removes=[],
date=datetime.utcnow() - timedelta(minutes=2))
# make sure there's exactly one bulk save recorded
self.assertEqual(len(bulk_save_attachments), 1)
# make sure none of the attachments were re-saved in rebuild
self.assertEqual(
[key for key, value in bulk_save_attachments[0].items()
if value.get('data')], [])
def archive_forms(domain, user, uploaded_data):
response = {"errors": [], "success": []}
form_ids = [row.get("form_id") for row in uploaded_data]
missing_forms = set(form_ids)
for xform_doc in iter_docs(XFormInstance.get_db(), form_ids):
xform = XFormInstance.wrap(xform_doc)
missing_forms.discard(xform["_id"])
if xform["domain"] != domain:
response["errors"].append(
_(u"XFORM {form_id} does not belong to domain {domain}").format(
form_id=xform["_id"], domain=xform["domain"]
)
)
continue
xform_string = _(u"XFORM {form_id} for domain {domain} by user '{username}'").format(
form_id=xform["_id"], domain=xform["domain"], username=user.username
)
try:
xform.archive(user=user.username)
response["success"].append(_(u"Successfully archived {form}").format(form=xform_string))
except Exception as e:
response["errors"].append(_(u"Could not archive {form}: {error}").format(form=xform_string, error=e))
for missing_form_id in missing_forms:
response["errors"].append(_(u"Could not find XForm {form_id}").format(form_id=missing_form_id))
return response
def rows(self):
domain = self.report.filter_values['domain']
startdate = self.report.filter_values['startdate']
enddate = self.report.filter_values['enddate']
key_base = 'submission xmlns user'
# todo this will do one couch view hit per relevant user. could be optimized to sql or something if desired
user_ids = self.report.get_user_ids()
rows = []
for user in user_ids:
last_submission = XFormInstance.get_db().view('all_forms/view',
startkey=[key_base, domain, self.xmlns, user, enddate],
endkey=[key_base, domain, self.xmlns, user, startdate],
limit=1,
reduce=False,
include_docs=True,
descending=True,
).one()
if last_submission:
wrapped = XFormInstance.wrap(last_submission['doc'])
user_row = [wrapped.get_data(path) for path in self.column_slugs]
else:
user_row = [NO_VALUE] * len(self.column_slugs)
rows.append((user, user_row))
# format
formatted_rows = list(self.report.format.format_output(rows))
# transpose
return [[_(col)] + [r[i] for r in formatted_rows] for i, col in enumerate(self.column_slugs)]
def delete_all_xforms(cls, domain=None, user_id=None):
view = 'couchforms/all_submissions_by_domain'
view_kwargs = {}
if domain and user_id:
view = 'reports_forms/all_forms'
view_kwargs = {
'startkey': ['submission user', domain, user_id],
'endkey': ['submission user', domain, user_id, {}],
}
elif domain:
view_kwargs = {
'startkey': [domain],
'endkey': [domain, {}]
}
cls._delete_all(
XFormInstance.get_db(),
view,
**view_kwargs
)
query = XFormInstanceSQL.objects
if domain is not None:
query = query.filter(domain=domain)
if user_id is not None:
query = query.filter(user_id=user_id)
query.all().delete()
def test_case_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
self._save_doc_to_db('indicator_form.json', XFormInstance)
case_id = self._save_doc_to_db('indicator_case.json', CommCareCase)
case_instance = CommCareCase.get(case_id)
# FormDataInCaseIndicatorDef (For those forgotten properties)
forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
case_type='song_tag',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
forgotten_property.save()
self.case_pillow.process_changes(since=since, forever=False)
indicator_case = IndicatorCase.get(case_id)
self.assertEqual(indicator_case.get_id, case_instance.get_id)
self.assertNotEqual(
indicator_case.get_db().dbname, case_instance.get_db().dbname
)
self.assertNotEqual(indicator_case.computed_, {})
def get_main_cache_invalidation_pillow(pillow_id, **kwargs):
from couchforms.models import XFormInstance
return _get_cache_invalidation_pillow(pillow_id,
XFormInstance.get_db(),
couch_filter="hqadmin/not_case_form")
def _clear_docs(self):
config = ExportConfiguration(
XFormInstance.get_db(),
[DOMAIN, "http://www.commcarehq.org/export/test"])
for form in config.get_docs():
XFormInstance.wrap(form).delete()
def tearDownClass(cls):
CommCareCase.get_db().bulk_delete(cls.cases)
XFormInstance.get_db().bulk_delete(cls.forms)
super(DBAccessorsTest, cls).tearDownClass()
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def get(self, *args, **kwargs):
"""
Download prior progress note submissions for local access
"""
db = XFormInstance.get_db()
couch_user = CouchUser.from_django_user(self.request.user)
username = couch_user.raw_username
if hasattr(localsettings, 'debug_pact_user'):
username = getattr(localsettings, 'debug_pact_user')(username)
offset = 0
limit_count = 200
total_count = 0
query = {
"query": {
"filtered": {
"filter": {
"and": [{
"term": {
"domain.exact": "pact"
}
}, {
"term": {
"form.#type": "progress_note"
}
}, {
"term": {
"form.meta.username": username
}
}]
},
"query": {
"match_all": {}
}
}
},
"sort": {
"received_on": "asc"
},
"size": limit_count,
"fields": ['_id', 'external_blobs']
}
query['script_fields'] = {}
query['script_fields'].update(pact_script_fields())
query['script_fields'].update(case_script_field())
res = self.xform_es.run_query(query)
my_patients_ever_submitted_query = query_per_case_submissions_facet(
PACT_DOMAIN, username)
patients_res = self.xform_es.run_query(
my_patients_ever_submitted_query)
#filter by active/discharged?
#get all the forms
#get all the patients
#get all patients to determine which to filter.
active_patients = []
for pt in []:
#if pt.hp_status == "Discharged":
#continue
case_id = pt['script_case_id']
active_patients.append(case_id)
def return_iterator():
yield "<restoredata>"
for result in res['hits']['hits']:
data_row = result['fields']
# if data_row['script_case_id'] not in active_patients:
# continue
try:
xml_str = (BlobHelper(
data_row, db,
CODES.form_xml).fetch_attachment('form.xml').replace(
"<?xml version=\'1.0\' ?>", '').replace(
"<?xml version='1.0' encoding='UTF-8' ?>", ''))
yield xml_str
except Exception as ex:
logging.error(
"for downloader: error fetching attachment: %s" % ex)
yield "</restoredata>"
response = HttpResponse(return_iterator(), content_type='text/xml')
return response
def delete_all_xforms():
_delete_all(XFormInstance.get_db(), 'couchforms/all_submissions_by_domain')
assert getattr(settings, 'UNIT_TESTING', False)
domain = get_and_check_xform_domain(xform)
with CaseDbCacheCouch(domain=domain, lock=True,
deleted_ok=True) as case_db:
case_result = process_cases_with_casedb([xform],
case_db,
config=config)
cases = case_result.cases
docs = [xform] + cases
now = datetime.datetime.utcnow()
for case in cases:
case.server_modified_on = now
XFormInstance.get_db().bulk_save(docs)
for case in cases:
case_post_save.send(CommCareCase, case=case)
case_result.commit_dirtiness_flags()
return cases
def reprocess_form_cases(form):
"""
For a given form, reprocess all case elements inside it. This operation
should be a no-op if the form was sucessfully processed, but should
correctly inject the update into the case history if the form was NOT
successfully processed.
"""
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
try:
tasks = filter(lambda x: x['type'] == "indexer",
db.server.active_tasks())
except Unauthorized:
return json_response(
{'error': "Unable to access CouchDB Tasks (unauthorized)."},
status_code=500)
if not is_bigcouch():
return json_response(tasks)
else:
# group tasks by design doc
task_map = defaultdict(dict)
for task in tasks:
meta = task_map[task['design_document']]
tasks = meta.get('tasks', [])
tasks.append(task)
meta['tasks'] = tasks
design_docs = []
for dd, meta in task_map.items():
meta['design_document'] = dd[len('_design/'):]
total_changes = sum(task['total_changes']
for task in meta['tasks'])
for task in meta['tasks']:
task['progress_contribution'] = task[
'changes_done'] * 100 / total_changes
design_docs.append(meta)
return json_response(design_docs)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == 'pillowtop':
pillow_meta = get_all_pillows_json()
supervisor_status = all_pillows_supervisor_status(
[meta['name'] for meta in pillow_meta])
for meta in pillow_meta:
meta.update(supervisor_status[meta['name']])
return json_response(
sorted(pillow_meta, key=lambda m: m['name'].lower()))
elif type == 'stale_pillows':
es_index_status = [
escheck.check_case_es_index(interval=3),
escheck.check_xform_es_index(interval=3),
escheck.check_reportcase_es_index(interval=3),
escheck.check_reportxform_es_index(interval=3)
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks",
params_dict={
'limit': task_limit
}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
return json_response(
{
'error':
"Error with getting from celery_flower: %s" % ex
},
status_code=500)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret),
content_type='application/json')
def soft_undelete_forms(domain, form_ids):
return _soft_undelete(XFormInstance.get_db(), form_ids)
def tearDownClass(cls):
CommCareCase.get_db().bulk_delete(cls.cases)
XFormInstance.get_db().bulk_delete(cls.forms)
e)
# one wonders if this will eventually have to paginate
domains = Domain.get_all()
from corehq.apps.domain.calculations import _all_domain_stats
all_stats = _all_domain_stats()
for dom in domains:
dom.web_users = int(all_stats["web_users"][dom.name])
dom.commcare_users = int(all_stats["commcare_users"][dom.name])
dom.cases = int(all_stats["cases"][dom.name])
dom.forms = int(all_stats["forms"][dom.name])
if dom.forms:
try:
dom.first_submission = string_to_datetime(XFormInstance.get_db().view\
("couchforms/all_submissions_by_domain",
reduce=False, limit=1,
startkey=[dom.name, "by_date"],
endkey=[dom.name, "by_date", {}]).all()[0]["key"][2]).strftime("%Y-%m-%d")
except Exception:
dom.first_submission = ""
try:
dom.last_submission = string_to_datetime(XFormInstance.get_db().view\
("couchforms/all_submissions_by_domain",
reduce=False, limit=1, descending=True,
startkey=[dom.name, "by_date", {}],
endkey=[dom.name, "by_date"]).all()[0]["key"][2]).strftime("%Y-%m-%d")
except Exception:
dom.last_submission = ""
else:
dom.first_submission = ""
def delete_all_xforms():
# handle with care
_delete_all(XFormInstance.get_db(),
'case/by_xform_id',
id_func=lambda row: row['key'])
def delete_all_xforms(cls, domain=None):
logger.debug("Deleting all Couch xforms for domain %s", domain)
cls._delete_all(XFormInstance.get_db(), all_known_formlike_doc_types(),
domain)
FormProcessorTestUtils.delete_all_sql_forms(domain)
class Command(BaseCommand):
args = '<id>'
help = ('''
Comprehensive reprocessing command for xforms that had post-submission
signaling errors that may not have updated the requisite case they were
attached to''')
option_list = LabelCommand.option_list + \
(
make_option('--report',
action='store_true',
dest='do_report',
default=False,
help="Analyze and print a report of the data"),
make_option('--from_date',
action='store',
dest='from_date',
default="",
help="Date to begin query range from"),
)
def println(self, message):
self.stdout.write("%s\n" % message)
def printerr(self, message):
self.stderr.write("%s\n" % message)
def get_all_domains(self):
db = Domain.get_db()
return [x['key'] for x in db.view('domain/domains', reduce=False).all()]
def get_all_submissions(self, domain, from_date):
#/receiverwrapper/_view/all_submissions_by_domain?startkey=["tulasalud","by_date","2013-03-07"]&endkey=["tulasalud","by_date","2013-05-07"]&reduce=false
db = XFormInstance.get_db()
chunk = 500
start = 0
sk = [domain, "by_date", from_date.strftime("%Y-%m-%d")]
ek = [domain, "by_date", (datetime.utcnow() + timedelta(days=10)).strftime("%Y-%m%d")]
def call_view(sk, ek, skip, limit):
return db.view('receiverwrapper/all_submissions_by_domain',
startkey=sk,
endkey=ek,
reduce=False,
limit=limit,
skip=skip,
include_docs=True
)
view_chunk = call_view(sk, ek, start, chunk)
while len(view_chunk) > 0:
for item in view_chunk:
if item['doc'] is not None:
yield item['doc']
start += chunk
view_chunk = call_view(sk, ek, start, chunk)
def is_case_updated(self, submission, method="couch"):
# use the same case processing utilities the case code does
def _case_ids_in_couch(submission):
case_view = CommCareCase.get_db().view('case/by_xform_id',
key=submission['_id'],
reduce=False).all()
return [row['id'] for row in case_view]
def _case_ids_in_es(submission):
query = {
"filter": {
"and": [
{"term": {"xform_ids": submission['_id']}}
]
},
"from": 0,
"size":1
}
es_results = self.es['hqcases'].post('_search', data=query)
return [row['_source']['_id'] for row in es_results['hits']['hits']] \
if es_results['hits']['hits'] else []
case_blocks = extract_case_blocks(submission)
case_updates = [case_update_from_block(case_block) for case_block in case_blocks]
case_ids_in_form = set(cu.id for cu in case_updates)
case_ids_in_db = set({
"couch": _case_ids_in_couch,
"es": _case_ids_in_es,
}[method](submission))
missing = case_ids_in_form - case_ids_in_db
return list(case_ids_in_form), list(missing), bool(missing)
def handle(self, *args, **options):
self.es = get_es()
try:
from_date = datetime.strptime(options['from_date'], "%Y-%m-%d")
except Exception, ex:
self.printerr("need a valid date string --from_date YYYY-mm-dd: %s" % ex)
sys.exit()
self.println(','.join(HEADERS))
domains = self.get_all_domains()
for ix, domain in enumerate(domains):
self.printerr("Domain: %s (%d/%d)" % (domain, ix, len(domains)))
for submit in self.get_all_submissions(domain, from_date):
outrow = [domain, submit['received_on'], submit['doc_type'], submit['_id']]
# basic case info
is_dupe=False
if submit['doc_type'] == 'XFormDuplicate':
is_dupe=True
orig_submit = XFormInstance.get_db().get(submit['form']['meta']['instanceID'])
case_ids, missing, updated = self.is_case_updated(orig_submit)
else:
case_ids, missing, updated = self.is_case_updated(submit)
if case_ids:
outrow.append("|".join(case_ids))
outrow.append("|".join(missing))
outrow.append(updated)
else:
outrow.append("nocase")
outrow.append("|".join(missing)) # would be weird if something was here
outrow.append("no update")
def _should_write():
# if we want to make this more configurable can adjust
# for now only output if there's a missing case and if
# it's not a dupe/deprecated
return missing and submit['doc_type'] not in ['XFormDeprecated', 'XFormDuplicate']
if _should_write():
self.println(','.join(str(x) for x in outrow))
def __init__(self, **kwargs):
super(UnknownUsersPillow, self).__init__(**kwargs)
self.couch_db = XFormInstance.get_db()
self.user_db = CouchUser.get_db()
self.es = get_es()
def setUp(self):
super(ChangeFeedDbTest, self).setUp()
self.couch_db = XFormInstance.get_db()
self.update_seq = get_current_seq(self.couch_db)
def print_stats(self, domain, short=True, diffs_only=False):
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"CommCareCase", (0, 0))
has_diffs |= self._print_status('CommCareCase', case_ids_in_couch,
case_ids_in_sql, diff_count,
num_docs_with_diffs, short, diffs_only)
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status('CommCareCase-Deleted',
case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def tearDown(self):
try:
XFormInstance.get_db().delete_doc("7H46J37FGH3")
except:
pass
def soft_delete_forms(domain,
form_ids,
deletion_date=None,
deletion_id=None):
return _soft_delete(XFormInstance.get_db(), form_ids, deletion_date,
deletion_id)
def get_all_forms_in_all_domains():
return [
XFormInstance.wrap(doc) for doc in get_all_docs_with_doc_types(
XFormInstance.get_db(), ['XFormInstance'])
]
def assign_new_id(cls, xform):
assert not xform.persistent_blobs, "some blobs would be lost"
new_id = XFormInstance.get_db().server.next_uuid()
xform._id = new_id
return xform
def tearDown(self):
view_kwargs = {
'startkey': ['submission user', TEST_DOMAIN, self.user1._id],
'endkey': ['submission user', TEST_DOMAIN, self.user1._id, {}],
}
FormProcessorTestUtils._delete_all_from_view(XFormInstance.get_db(), 'all_forms/view', view_kwargs)
def get_doc_count(model_class, where, entity, domain):
sql_estimate = estimate_partitioned_row_count(model_class, where)
couchdb = XFormInstance.get_db()
couch_count = get_couch_doc_count(domain, entity, couchdb)
return min(sql_estimate, couch_count)
def process_indicators_for_case(namespaces, domain, doc_dict):
case_type = doc_dict.get('type')
if not case_type:
return
case_indicator_defs = []
for namespace in namespaces:
case_indicator_defs.extend(
CaseIndicatorDefinition.get_all(namespace,
domain,
case_type=case_type))
try:
indicator_case = IndicatorCase.wrap_for_indicator_db(doc_dict)
indicator_case.update_indicators_in_bulk(case_indicator_defs,
logger=pillow_logging,
save_on_update=False)
indicator_case.save()
except Exception as e:
pillow_logging.error(
"Error creating for MVP Indicator for form %(form_id)s: "
"%(error)s" % {
'form_id': doc_dict['_id'],
'error': e,
}, )
raise
# Now Update Data From Case to All Related Xforms (ewwww)
xform_ids = doc_dict.get('xform_ids', [])
if not xform_ids:
return
for xform_id in xform_ids:
related_xform_indicators = []
try:
# first try to get the doc from the indicator DB
xform_doc = IndicatorXForm.get(xform_id)
except ResourceNotFound:
# if that fails fall back to the main DB
try:
xform_dict = XFormInstance.get_db().get(xform_id)
xform_doc = IndicatorXForm.wrap_for_indicator_db(xform_dict)
related_xform_indicators = get_form_indicators(
namespaces, domain, xform_doc.xmlns)
except ResourceNotFound:
pillow_logging.error(
"Could not find an XFormInstance with id %(xform_id)s "
"related to Case %(case_id)s" % {
'xform_id': xform_id,
'case_id': doc_dict['_id'],
})
continue
if not xform_doc.xmlns:
continue
for namespace in namespaces:
related_xform_indicators.extend(
CaseDataInFormIndicatorDefinition.get_all(
namespace, domain, xmlns=xform_doc.xmlns))
xform_doc.update_indicators_in_bulk(related_xform_indicators,
logger=pillow_logging,
save_on_update=False)
xform_doc.save()
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(
re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def reprocess_xform_error(form):
"""
Attempt to re-process an error form. This was created specifically to address
the issue of out of order forms and child cases (form creates child case before
parent case has been created).
See http://manage.dimagi.com/default.asp?250459
:param form_id: ID of the error form to process
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.submission_post import SubmissionPost
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL, FormAccessorSQL, LedgerAccessorSQL
from corehq.blobs.mixin import bulk_atomic_blobs
from couchforms.models import XFormInstance
from casexml.apps.case.signals import case_post_save
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
if not form:
raise Exception('Form with ID {} not found'.format(form.form_id))
if not form.is_error:
raise Exception('Form was not an error form: {}={}'.format(form.form_id, form.doc_type))
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
form.initial_processing_complete = True
form.problem = None
cache = FormProcessorInterface(form.domain).casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form]
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
if case_stock_result:
stock_result = case_stock_result.stock_result
if stock_result:
assert stock_result.populated
cases = case_stock_result.case_models
if should_use_sql_backend(form.domain):
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
LedgerAccessorSQL.save_ledger_values(stock_result.models_to_save)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None),
cases,
stock_result
)
else:
with bulk_atomic_blobs([form] + cases):
XFormInstance.save(form) # use this save to that we don't overwrite the doc_type
XFormInstance.get_db().bulk_save(cases)
if stock_result:
stock_result.commit()
case_stock_result.stock_result.finalize()
case_stock_result.case_result.commit_dirtiness_flags()
for case in cases:
case_post_save.send(case.__class__, case=case)
return form
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild upated ledgers
interface.ledger_processor.hard_rebuild_ledgers(
**ledger.ledger_reference._asdict())
else:
if save:
with bulk_atomic_blobs([form] + cases):
XFormInstance.save(
form
) # use this save to that we don't overwrite the doc_type
XFormInstance.get_db().bulk_save(cases)
stock_result.commit()
save and case_stock_result.stock_result.finalize()
save and case_stock_result.case_result.commit_dirtiness_flags()
return ReprocessingResult(form, cases, ledgers)
def _log_changes(slug, cases, stock_updates, stock_deletes):
if logger.isEnabledFor(logging.INFO):
case_ids = [case.case_id for case in cases]
logger.info(
"%s changes:\n\tcases: %s\n\tstock changes%s\n\tstock deletes%s",
slug, case_ids, stock_updates, stock_deletes)
def soft_undelete_forms(domain, form_ids):
def _form_undelete(doc):
doc['server_modified_on'] = json_format_datetime(datetime.utcnow())
return _soft_undelete(XFormInstance.get_db(), form_ids, _form_undelete)
def commit_plan(domain, planning_db):
assert get_planning_db_filepath(domain) == planning_db.db_filepath
for form in planning_db.get_forms():
XFormInstance.get_db().save_doc(form)
for case in planning_db.get_cases():
CommCareCase.get_db().save_doc(case)
def save(self, **kwargs):
self.doc_type = 'IndicatorXForm'
assert self.get_db().uri != XFormInstance.get_db().uri
super(IndicatorXForm, self).save(**kwargs)
