def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain,
for_comparing=True)
case_cache = CaseDbCache(domain=self.domain,
strip_history=False,
deleted_ok=True)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (
not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(
message.couch_recipient).get_case_property("study_arm")
timestamp = ServerTime(message.date).user_time(
self.timezone).done()
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction, "-")),
])
return result
def test_wrap_lock_dependency(self):
# valid combinations
CaseDbCache(domain='some-domain', lock=False, wrap=True)
CaseDbCache(domain='some-domain', lock=False, wrap=False)
CaseDbCache(domain='some-domain', lock=True, wrap=True)
with self.assertRaises(ValueError):
# invalid
CaseDbCache(domain='some-domain', lock=True, wrap=False)
def testGetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = CaseDbCache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i), case.my_index)
def get_related_cases(initial_cases, domain, strip_history=False, search_up=True):
"""
Gets the flat list of related cases based on a starting list.
Walks all the referenced indexes recursively.
If search_up is True, all cases and their parent cases are returned.
If search_up is False, all cases and their child cases are returned.
"""
if not initial_cases:
return {}
# infer whether to wrap or not based on whether the initial list is wrapped or not
# initial_cases may be a list or a set
wrap = isinstance(next(iter(initial_cases)), CommCareCase)
# todo: should assert that domain exists here but this breaks tests
case_db = CaseDbCache(domain=domain,
strip_history=strip_history,
deleted_ok=True,
wrap=wrap,
initial=initial_cases)
def indices(case):
return case['indices'] if search_up else reverse_indices(CommCareCase.get_db(), case, wrap=False)
relevant_cases = {}
relevant_deleted_case_ids = []
cases_to_process = list(case for case in initial_cases)
directly_referenced_indices = itertools.chain(
*[[index['referenced_id'] for index in indices(case)]
for case in initial_cases]
)
case_db.populate(directly_referenced_indices)
def process_cases(cases):
new_relations = set()
for case in cases:
if case and case['_id'] not in relevant_cases:
relevant_cases[case['_id']] = case
if case['doc_type'] == 'CommCareCase-Deleted':
relevant_deleted_case_ids.append(case['_id'])
new_relations.update(index['referenced_id'] for index in indices(case))
if new_relations:
case_db.populate(new_relations)
return [case_db.get(related_case) for related_case in new_relations]
while cases_to_process:
cases_to_process = process_cases(cases_to_process)
if relevant_deleted_case_ids:
logging.info('deleted cases included in footprint (restore): %s' % (
', '.join(relevant_deleted_case_ids)
))
return relevant_cases
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain, for_comparing=True)
case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm")
timestamp = ServerTime(message.date).user_time(self.timezone).done()
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction,"-")),
])
return result
def testGetPopulatesCache(self):
case_ids = _make_some_cases(3)
cache = CaseDbCache()
for id in case_ids:
self.assertFalse(cache.in_cache(id))
for i, id in enumerate(case_ids):
case = cache.get(id)
self.assertEqual(str(i), case.my_index)
for id in case_ids:
self.assertTrue(cache.in_cache(id))
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
cache = CaseDbCache()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
def testDocTypeCheck(self):
id = uuid.uuid4().hex
CommCareCase.get_db().save_doc({
"_id": id,
"doc_type": "AintNoCasesHere"
})
cache = CaseDbCache()
try:
cache.get(id)
self.fail('doc type security check failed to raise exception')
except IllegalCaseId:
pass
doc_back = CommCareCase.get_db().get(id)
self.assertEqual("AintNoCasesHere", doc_back['doc_type'])
def get_related_cases(initial_cases, domain, strip_history=False, search_up=True):
"""
Gets the flat list of related cases based on a starting list.
Walks all the referenced indexes recursively.
If search_up is True, all cases and their parent cases are returned.
If search_up is False, all cases and their child cases are returned.
"""
if not initial_cases:
return {}
# infer whether to wrap or not based on whether the initial list is wrapped or not
# initial_cases may be a list or a set
wrap = isinstance(next(iter(initial_cases)), CommCareCase)
# todo: should assert that domain exists here but this breaks tests
case_db = CaseDbCache(domain=domain,
strip_history=strip_history,
deleted_ok=True,
wrap=wrap,
initial=initial_cases)
def indices(case):
return case['indices'] if search_up else get_reverse_indices_json(case)
relevant_cases = {}
relevant_deleted_case_ids = []
cases_to_process = list(case for case in initial_cases)
def get_related_cases(initial_case_list,
domain,
strip_history=False,
search_up=True):
"""
Gets the flat list of related cases based on a starting list.
Walks all the referenced indexes recursively.
If search_up is True, all cases and their parent cases are returned.
If search_up is False, all cases and their child cases are returned.
"""
if not initial_case_list:
return {}
# todo: should assert that domain exists here but this breaks tests
case_db = CaseDbCache(domain=domain,
strip_history=strip_history,
deleted_ok=True)
def related(case_db, case):
return [
case_db.get(index.referenced_id)
for index in (case.indices if search_up else case.reverse_indices)
]
relevant_cases = {}
relevant_deleted_case_ids = []
queue = list(case for case in initial_case_list)
def test(self):
form = make_form_from_case_blocks([ElementTree.fromstring(CASE_BLOCK)])
lock_manager = process_xform(form, domain=self.domain)
with lock_manager as xforms:
with CaseDbCache(domain=self.domain) as case_db:
with self.assertRaises(PhoneDateValueError):
process_cases_with_casedb(xforms, case_db)
def process_stock(xform, case_db=None):
"""
process the commtrack xml constructs in an incoming submission
"""
case_db = case_db or CaseDbCache()
assert isinstance(case_db, CaseDbCache)
stock_report_helpers, case_actions = get_stock_actions(xform)
# omitted: normalize_transactions (used for bulk requisitions?)
# validate product ids
if any(transaction_helper.product_id in ('', None)
for stock_report_helper in stock_report_helpers
for transaction_helper in stock_report_helper.transactions):
raise MissingProductId(
_('Product IDs must be set for all ledger updates!'))
relevant_cases = []
# touch every case for proper ota restore logic syncing to be preserved
for case_id, case_action in case_actions:
case = case_db.get(case_id)
relevant_cases.append(case)
if case is None:
raise IllegalCaseId(
_('Ledger transaction references invalid Case ID "{}"')
.format(case_id))
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
case_action.sync_log_id = ''
case.actions.append(case_action)
case_db.mark_changed(case)
def testDomainCheck(self):
id = uuid.uuid4().hex
post_case_blocks([
CaseBlock(
create=True, case_id=id,
user_id='some-user', version=V2
).as_xml()
], {'domain': 'good-domain'}
)
bad_cache = CaseDbCache(domain='bad-domain')
try:
bad_cache.get(id)
self.fail('domain security check failed to raise exception')
except IllegalCaseId:
pass
good_cache = CaseDbCache(domain='good-domain')
case = good_cache.get(id)
self.assertEqual('some-user', case.user_id) # just sanity check it's the right thing
def get_related_cases(initial_case_list, domain, strip_history=False, search_up=True):
"""
Gets the flat list of related cases based on a starting list.
Walks all the referenced indexes recursively.
If search_up is True, all cases and their parent cases are returned.
If search_up is False, all cases and their child cases are returned.
"""
if not initial_case_list:
return {}
# todo: should assert that domain exists here but this breaks tests
case_db = CaseDbCache(domain=domain,
strip_history=strip_history,
deleted_ok=True,
initial=initial_case_list)
def related(case_db, case):
return [case_db.get(index.referenced_id) for index in (case.indices if search_up else case.reverse_indices)]
relevant_cases = {}
relevant_deleted_case_ids = []
queue = list(case for case in initial_case_list)
directly_referenced_indices = itertools.chain(*[[index.referenced_id for index in (case.indices if search_up else case.reverse_indices)]
for case in initial_case_list])
case_db.populate(directly_referenced_indices)
while queue:
case = queue.pop()
if case and case.case_id not in relevant_cases:
relevant_cases[case.case_id] = case
if case.doc_type == 'CommCareCase-Deleted':
relevant_deleted_case_ids.append(case.case_id)
queue.extend(related(case_db, case))
if relevant_deleted_case_ids:
logging.info('deleted cases included in footprint (restore): %s' % (
', '.join(relevant_deleted_case_ids)
))
return relevant_cases
def testDomainCheck(self):
id = uuid.uuid4().hex
post_case_blocks([
CaseBlock(create=True, case_id=id, user_id='some-user',
version=V2).as_xml()
], {'domain': 'good-domain'})
bad_cache = CaseDbCache(domain='bad-domain')
try:
bad_cache.get(id)
self.fail('domain security check failed to raise exception')
except IllegalCaseId:
pass
good_cache = CaseDbCache(domain='good-domain')
case = good_cache.get(id)
self.assertEqual(
'some-user',
case.user_id) # just sanity check it's the right thing
class TestCasesReceivedSignal(TestCase):
def test_casedb_already_has_cases(self):
case = CaseFactory().create_case()
case_db = CaseDbCache(
initial=[CommCareCase(_id='fake1'),
CommCareCase(_id='fake2')])
form = XFormInstance.get(case.xform_ids[0])
def assert_exactly_one_case(sender, xform, cases, **kwargs):
global case_count
case_count = len(cases)
cases_received.connect(assert_exactly_one_case)
try:
process_cases_with_casedb([form], case_db)
self.assertEqual(1, case_count)
finally:
cases_received.disconnect(assert_exactly_one_case)
def testStripHistory(self):
case_ids = _make_some_cases(3)
history_cache = CaseDbCache()
for i, id in enumerate(case_ids):
self.assertFalse(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
nohistory_cache = CaseDbCache(strip_history=True)
for i, id in enumerate(case_ids):
self.assertFalse(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
more_case_ids = _make_some_cases(3)
history_cache.populate(more_case_ids)
nohistory_cache.populate(more_case_ids)
for i, id in enumerate(more_case_ids):
self.assertTrue(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
for i, id in enumerate(more_case_ids):
self.assertTrue(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = CaseDbCache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))
def testStripHistory(self):
case_ids = _make_some_cases(3)
history_cache = CaseDbCache()
for i, id in enumerate(case_ids):
self.assertFalse(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
nohistory_cache = CaseDbCache(strip_history=True)
for i, id in enumerate(case_ids):
self.assertFalse(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
more_case_ids = _make_some_cases(3)
history_cache.populate(more_case_ids)
nohistory_cache.populate(more_case_ids)
for i, id in enumerate(more_case_ids):
self.assertTrue(history_cache.in_cache(id))
case = history_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) > 0)
for i, id in enumerate(more_case_ids):
self.assertTrue(nohistory_cache.in_cache(id))
case = nohistory_cache.get(id)
self.assertEqual(str(i), case.my_index)
self.assertTrue(len(case.actions) == 0)
def run(self):
if not self.auth_context.is_valid():
return self.failed_auth_response, None, []
if isinstance(self.instance, const.BadRequest):
return HttpResponseBadRequest(self.instance.message), None, []
def process(xform):
self._attach_shared_props(xform)
scrub_meta(xform)
try:
lock_manager = process_xform(self.instance,
attachments=self.attachments,
process=process,
domain=self.domain)
except SubmissionError as e:
logging.exception(
u"Problem receiving submission to %s. %s" % (
self.path,
unicode(e),
)
)
return self.get_exception_response(e.error_log), None, []
else:
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.xform import (
get_and_check_xform_domain, CaseDbCache, process_cases_with_casedb
)
from casexml.apps.case.signals import case_post_save
from casexml.apps.case.exceptions import IllegalCaseId, UsesReferrals
from corehq.apps.commtrack.processing import process_stock
from corehq.apps.commtrack.exceptions import MissingProductId
cases = []
responses = []
errors = []
known_errors = (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError)
with lock_manager as xforms:
instance = xforms[0]
if instance.doc_type == 'XFormInstance':
if len(xforms) > 1:
assert len(xforms) == 2
assert is_deprecation(xforms[1])
domain = get_and_check_xform_domain(instance)
with CaseDbCache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db:
try:
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(instance, case_db)
except known_errors as e:
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
instance = _handle_known_error(e, instance)
xforms[0] = instance
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
XFormInstance.get_db().bulk_save(xforms)
response = self._get_open_rosa_response(instance,
None, None)
return response, instance, cases
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
error_message = u'{}: {}'.format(type(e).__name__, unicode(e))
instance = _handle_unexpected_error(instance, error_message)
instance.save()
raise
now = datetime.datetime.utcnow()
unfinished_submission_stub = UnfinishedSubmissionStub(
xform_id=instance.get_id,
timestamp=now,
saved=False,
domain=domain,
)
unfinished_submission_stub.save()
cases = case_db.get_changed()
# todo: this property is useless now
instance.initial_processing_complete = True
assert XFormInstance.get_db().uri == CommCareCase.get_db().uri
docs = xforms + cases
# in saving the cases, we have to do all the things
# done in CommCareCase.save()
for case in cases:
case.initial_processing_complete = True
case.server_modified_on = now
try:
rev = CommCareCase.get_db().get_rev(case.case_id)
except ResourceNotFound:
pass
else:
assert rev == case.get_rev, (
"Aborting because there would have been "
"a document update conflict. {} {} {}".format(
case.get_id, case.get_rev, rev
)
)
try:
# save both the forms and cases
XFormInstance.get_db().bulk_save(docs)
except BulkSaveError as e:
logging.error('BulkSaveError saving forms', exc_info=1,
extra={'details': {'errors': e.errors}})
raise
except Exception as e:
docs_being_saved = [doc['_id'] for doc in docs]
error_message = u'Unexpected error bulk saving docs {}: {}, doc_ids: {}'.format(
type(e).__name__,
unicode(e),
', '.join(docs_being_saved)
)
instance = _handle_unexpected_error(instance, error_message)
instance.save()
raise
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
case_result.commit_dirtiness_flags()
stock_result.commit()
for case in cases:
case_post_save.send(CommCareCase, case=case)
responses, errors = self.process_signals(instance)
if errors:
# .problems was added to instance
instance.save()
unfinished_submission_stub.delete()
elif instance.doc_type == 'XFormDuplicate':
assert len(xforms) == 1
instance.save()
def test_nowrap(self):
case_ids = _make_some_cases(1)
cache = CaseDbCache(wrap=False)
case = cache.get(case_ids[0])
self.assertTrue(isinstance(case, dict))
self.assertFalse(isinstance(case, CommCareCase))