class TestHardDelete(TestCase):
def setUp(self):
self.casedb = CaseAccessors(TEST_DOMAIN_NAME)
self.formdb = FormAccessors(TEST_DOMAIN_NAME)
@run_with_all_backends
def test_simple_delete(self):
factory = CaseFactory()
case = factory.create_case()
[case] = factory.create_or_update_case(
CaseStructure(case_id=case.case_id, attrs={'update': {'foo': 'bar'}})
)
self.assertIsNotNone(self.casedb.get_case(case.case_id))
self.assertEqual(2, len(case.xform_ids))
for form_id in case.xform_ids:
self.assertIsNotNone(self.formdb.get_form(form_id))
safe_hard_delete(case)
with self.assertRaises(CaseNotFound):
self.casedb.get_case(case.case_id)
for form_id in case.xform_ids:
with self.assertRaises(XFormNotFound):
self.formdb.get_form(form_id)
@run_with_all_backends
def test_delete_with_related(self):
factory = CaseFactory()
parent = factory.create_case()
[child] = factory.create_or_update_case(
CaseStructure(attrs={'create': True}, walk_related=False, indices=[
CaseIndex(CaseStructure(case_id=parent.case_id))
]),
)
# deleting the parent should not be allowed because the child still references it
with self.assertRaises(CommCareCaseError):
safe_hard_delete(parent)
# deleting the child is ok
safe_hard_delete(child)
self.assertIsNotNone(self.casedb.get_case(parent.case_id))
with self.assertRaises(CaseNotFound):
self.casedb.get_case(child.case_id)
@run_with_all_backends
def test_delete_sharing_form(self):
factory = CaseFactory()
c1, c2 = factory.create_or_update_cases([
CaseStructure(attrs={'create': True}),
CaseStructure(attrs={'create': True}),
])
with self.assertRaises(CommCareCaseError):
safe_hard_delete(c1)
with self.assertRaises(CommCareCaseError):
safe_hard_delete(c2)
self.assertIsNotNone(self.casedb.get_case(c1.case_id))
self.assertIsNotNone(self.casedb.get_case(c2.case_id))
def handle(self, domain, folder_path, **options):
if os.path.exists(folder_path):
if not os.path.isdir(folder_path):
raise CommandError('Folder path must be the path to a directory')
else:
os.mkdir(folder_path)
form_accessors = FormAccessors(domain)
form_ids = form_accessors.get_all_form_ids_in_domain()
for form in form_accessors.iter_forms(form_ids):
form_path = os.path.join(folder_path, form.form_id)
if not os.path.exists(form_path):
os.mkdir(form_path)
form_meta = FormMetadata(
user_id=form.user_id,
received_on=form.received_on,
app_id=form.app_id,
build_id=form.build_id,
attachments=form.attachments.keys(),
auth_context=form.auth_context,
)
with open(os.path.join(form_path, 'metadata.json'), 'w') as meta:
json.dump(form_meta.to_json(), meta)
xml = form.get_xml()
with open(os.path.join(form_path, 'form.xml'), 'w') as f:
f.write(xml)
for name, meta in form.attachments.items():
with open(os.path.join(form_path, name), 'w') as f:
f.write(form.get_attachment(name))
def test_submit_deprecated_form_with_attachments(self):
def list_attachments(form):
return sorted(
(att.name, att.open().read())
for att in form.get_attachments()
if att.name != "form.xml"
)
self._submit('simple_form.xml', attachments={
"image": BytesIO(b"fake image"),
"file": BytesIO(b"text file"),
})
response = self._submit(
'simple_form_edited.xml',
attachments={"image": BytesIO(b"other fake image")},
url=reverse("receiver_secure_post", args=[self.domain]),
)
acc = FormAccessors(self.domain.name)
new_form = acc.get_form(response['X-CommCareHQ-FormID'])
old_form = acc.get_form(new_form.deprecated_form_id)
self.assertIn(b"<bop>bang</bop>", old_form.get_xml())
self.assertIn(b"<bop>bong</bop>", new_form.get_xml())
self.assertEqual(list_attachments(old_form),
[("file", b"text file"), ("image", b"fake image")])
self.assertEqual(list_attachments(new_form),
[("file", b"text file"), ("image", b"other fake image")])
def _handle_duplicate(new_doc, instance):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
and *must* include inline attachments
"""
interface = FormProcessorInterface(new_doc.domain)
conflict_id = new_doc.form_id
existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id)
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
if existing_md5 != new_md5:
# if the form contents are not the same:
# - "Deprecate" the old form by making a new document with the same contents
# but a different ID and a doc_type of XFormDeprecated
# - Save the new instance to the previous document to preserve the ID
existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface)
return FormProcessingResult(new_doc, existing_doc)
else:
# follow standard dupe handling, which simply saves a copy of the form
# but a new doc_id, and a doc_type of XFormDuplicate
duplicate = interface.deduplicate_xform(new_doc)
return FormProcessingResult(duplicate)
def test_archiving_only_form(self):
"""
Checks that archiving the only form associated with the case archives
the case and unarchiving unarchives it.
"""
case_id = _post_util(create=True, p1='p1-1', p2='p2-1')
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
self.assertFalse(case.is_deleted)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(1, len(case.actions))
else:
self.assertEqual(2, len(case.actions))
[form_id] = case.xform_ids
form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id)
form.archive()
case = case_accessors.get_case(case_id)
self.assertTrue(case.is_deleted)
# should just have the 'rebuild' action
self.assertEqual(1, len(case.actions))
self.assertTrue(case.actions[0].is_case_rebuild)
form.unarchive()
case = case_accessors.get_case(case_id)
self.assertFalse(case.is_deleted)
self.assertEqual(3, len(case.actions))
self.assertTrue(case.actions[-1].is_case_rebuild)
def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = list(itertools.chain(*[
form_accessor.get_all_form_ids_in_domain(doc_type=doc_type)
for doc_type in doc_type_to_state
]))
for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def test_archive_last_form(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
self.submit_xml_form(
balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30)
)
final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)]
second_form_id = self.submit_xml_form(balance_submission(final_amounts))
ledger_accessors = LedgerAccessors(self.domain.name)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(
round(float(lv.daily_consumption), 2),
1.67
)
# check initial setup
_assert_initial_state()
# archive and confirm commtrack data is deleted
form = FormAccessors(self.domain.name).get_form(second_form_id)
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.archive()
if should_use_sql_backend(self.domain):
self.assertEqual(0, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(0, StockReport.objects.filter(form_id=second_form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
# balance should be reverted to 100 in the StockState
self.assertEqual(100, int(state.stock_on_hand))
# consumption should be none since there will only be 1 data point
self.assertIsNone(state.daily_consumption)
# unarchive and confirm commtrack data is restored
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.unarchive()
_assert_initial_state()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None):
assert domain, "'domain' is required"
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
if case_db:
assert case_db.domain == domain
self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3
self.track_load = form_load_counter("form_submission", domain)
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
def test_archived_form_gets_removed_from_case_xform_ids(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
instance_id = self.submit_xml_form(
balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30)
)
case_accessors = CaseAccessors(self.domain.name)
case = case_accessors.get_case(self.sp.case_id)
self.assertIn(instance_id, case.xform_ids)
form = FormAccessors(self.domain.name).get_form(instance_id)
form.archive()
case = case_accessors.get_case(self.sp.case_id)
self.assertNotIn(instance_id, case.xform_ids)
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
return self.form_accessors.get_form(doc_id).to_json()
def test_reprocess_xform_error(self):
case_id = uuid.uuid4().hex
parent_case_id = uuid.uuid4().hex
case = CaseBlock(
create=True,
case_id=case_id,
user_id='user1',
owner_id='user1',
case_type='demo',
case_name='child',
index={'parent': ('parent_type', parent_case_id)}
)
post_case_blocks([case.as_xml()], domain=self.domain)
form_accessors = FormAccessors(self.domain)
error_forms = form_accessors.get_forms_by_type('XFormError', 10)
self.assertEqual(1, len(error_forms))
form = error_forms[0]
reprocess_xform_error(form)
error_forms = form_accessors.get_forms_by_type('XFormError', 10)
self.assertEqual(1, len(error_forms))
case = CaseBlock(
create=True,
case_id=parent_case_id,
user_id='user1',
owner_id='user1',
case_type='parent_type',
case_name='parent',
)
post_case_blocks([case.as_xml()], domain=self.domain)
reprocess_xform_error(form_accessors.get_form(form.form_id))
form = form_accessors.get_form(form.form_id)
# self.assertTrue(form.initial_processing_complete) Can't change this with SQL forms at the moment
self.assertTrue(form.is_normal)
self.assertIsNone(form.problem)
case = CaseAccessors(self.domain).get_case(case_id)
self.assertEqual(1, len(case.indices))
self.assertEqual(case.indices[0].referenced_id, parent_case_id)
self._validate_case(case)
def test_soft_delete(self):
meta = TestFormMetadata(domain=DOMAIN)
get_simple_wrapped_form('f1', metadata=meta)
f2 = get_simple_wrapped_form('f2', metadata=meta)
f2.archive()
get_simple_wrapped_form('f3', metadata=meta)
accessors = FormAccessors(DOMAIN)
# delete
num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123')
self.assertEqual(num, 2)
for form_id in ['f1', 'f2']:
form = accessors.get_form(form_id)
self.assertTrue(form.is_deleted)
self.assertEqual(form.deletion_id, '123')
form = accessors.get_form('f3')
self.assertFalse(form.is_deleted)
for form_id in ['f1', 'f2']:
form = FormAccessors(DOMAIN).get_form(form_id)
if isinstance(form, DisabledDbMixin):
super(DisabledDbMixin, form).delete()
else:
form.delete()
def test_archive_against_deleted_case(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True, p1='p1', form_extras={'received_on': now})
_post_util(case_id=case_id, p2='p2',
form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id, p3='p3',
form_extras={'received_on': now + timedelta(seconds=2)})
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
case_accessors.soft_delete_cases([case_id])
[f1, f2, f3] = case.xform_ids
f2_doc = FormAccessors(REBUILD_TEST_DOMAIN).get_form(f2)
f2_doc.archive()
case = case_accessors.get_case(case_id)
self.assertTrue(case.is_deleted)
def test_error_with_normal_doc_type_migration(self):
submit_form_locally(
"""<data xmlns="example.com/foo">
<meta>
<instanceID>im-a-bad-form</instanceID>
</meta>
<case case_id="" xmlns="http://commcarehq.org/case/transaction/v2">
<update><foo>bar</foo></update>
</case>
</data>""",
self.domain_name,
)
form = FormAccessors(self.domain_name).get_form('im-a-bad-form')
form_json = form.to_json()
form_json['doc_type'] = 'XFormInstance'
XFormInstance.wrap(form_json).save()
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(1, len(self._get_form_ids('XFormError')))
self._compare_diffs([])
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
try:
return add_couch_properties_to_sql_form_json(self.form_accessors.get_form(doc_id).to_json())
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.get_all_form_ids_in_domain())
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield add_couch_properties_to_sql_form_json(wrapped_form.to_json())
def test_update_responses(self):
formxml = FormSubmissionBuilder(
form_id='123',
form_properties={'breakfast': 'toast', 'lunch': 'sandwich'}
).as_xml_string()
pic = UploadedFile(BytesIO(b"fake"), 'pic.jpg', content_type='image/jpeg')
xform = submit_form_locally(formxml, DOMAIN, attachments={"image": pic}).xform
updates = {'breakfast': 'fruit'}
errors = FormProcessorInterface(DOMAIN).update_responses(xform, updates, 'user1')
form = FormAccessors(DOMAIN).get_form(xform.form_id)
self.assertEqual(0, len(errors))
self.assertEqual('fruit', form.form_data['breakfast'])
self.assertEqual('sandwich', form.form_data['lunch'])
self.assertIn("image", form.attachments)
self.assertEqual(form.get_attachment("image"), b"fake")
self.assertXmlEqual(
form.get_attachment("form.xml").decode('utf-8'),
formxml.replace("toast", "fruit"),
)
def _reopen_or_create_supply_point(location):
from .helpers import update_supply_point_from_location
from .dbaccessors import get_supply_point_by_location_id
supply_point = get_supply_point_by_location_id(location.domain, location.location_id)
if supply_point:
if supply_point and supply_point.closed:
form_ids = CaseAccessors(supply_point.domain).get_case_xform_ids(supply_point.case_id)
form_accessor = FormAccessors(supply_point.domain)
for form_id in form_ids:
form = form_accessor.get_form(form_id)
closes_case = any(map(
lambda case_update: case_update.closes_case(),
get_case_updates(form),
))
if closes_case:
form.archive(user_id=const.COMMTRACK_USERNAME)
update_supply_point_from_location(supply_point, location)
return supply_point
else:
return SupplyInterface.create_from_location(location.domain, location)
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
try:
return self.form_accessors.get_form(doc_id).to_json()
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def perform_resave_on_xforms(domain, start_date, end_date, no_input):
_, _, xform_ids_missing_in_es, _ = compare_xforms(domain, 'XFormInstance', start_date, end_date)
print("%s Ids found for xforms missing in ES." % len(xform_ids_missing_in_es))
if len(xform_ids_missing_in_es) < 1000:
print(xform_ids_missing_in_es)
if no_input is not True:
ok = input("Type 'ok' to continue: ")
if ok != "ok":
print("No changes made")
return
form_accessor = FormAccessors(domain)
for xform_ids in chunked(with_progress_bar(xform_ids_missing_in_es), 100):
xforms = form_accessor.get_forms(list(xform_ids))
found_xform_ids = set()
for xform in xforms:
resave_form(domain, xform)
found_xform_ids.add(xform.form_id)
for xform_id in set(xform_ids) - found_xform_ids:
print("form not found %s" % xform_id)
def handle(self, username, domain, **options):
this_form_accessor = FormAccessors(domain=domain)
user = CouchUser.get_by_username(username)
if not user:
logger.info("User {} not found.".format(username))
sys.exit(1)
user_id = user._id
form_ids = this_form_accessor.get_form_ids_for_user(user_id)
input_response = six.moves.input(
"Update {} form(s) for user {} in domain {}? (y/n): ".format(len(form_ids), username, domain))
if input_response == "y":
for form_data in this_form_accessor.iter_forms(form_ids):
form_attachment_xml_new = self.update_form_data(form_data, NEW_USERNAME)
this_form_accessor.modify_attachment_xml_and_metadata(form_data,
form_attachment_xml_new,
NEW_USERNAME)
logging.info("Updated {} form(s) for user {} in domain {}".format(len(form_ids), username, domain))
elif input_response == "n":
logging.info("No forms updated, exiting.")
else:
logging.info("Command not recognized. Exiting.")
def test_form_with_missing_xmlns(self):
form_id = uuid.uuid4().hex
form_template = """<?xml version='1.0' ?>
<data uiVersion="1" version="1" name=""{xmlns}>
<name>fgg</name>
<n1:meta xmlns:n1="http://openrosa.org/jr/xforms">
<n1:deviceID>354957031935664</n1:deviceID>
<n1:timeStart>2016-03-01T12:04:16Z</n1:timeStart>
<n1:timeEnd>2016-03-01T12:04:16Z</n1:timeEnd>
<n1:username>bcdemo</n1:username>
<n1:userID>user-abc</n1:userID>
<n1:instanceID>{form_id}</n1:instanceID>
</n1:meta>
</data>"""
xml = form_template.format(
form_id=form_id,
xmlns=' xmlns="http://openrosa.org/formdesigner/456"'
)
submit_form_locally(xml, self.domain_name)
# hack the form to remove XMLNS since it's now validated during form submission
form = FormAccessors(self.domain_name).get_form(form_id)
form.xmlns = None
del form.form_data['@xmlns']
xml_no_xmlns = form_template.format(form_id=form_id, xmlns="")
form.delete_attachment('form.xml')
form.put_attachment(xml_no_xmlns, 'form.xml')
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(1, len(self._get_form_ids()))
self._compare_diffs([])
def handle(self, domains, file_name, **options):
blob_db = get_blob_db()
with open(file_name, 'w', encoding='utf-8') as csv_file:
field_names = ['domain', 'archived', 'form_id', 'received_on']
csv_writer = csv.DictWriter(csv_file, field_names)
csv_writer.writeheader()
for domain in domains:
self.stdout.write("Handling domain %s" % domain)
form_db = FormAccessors(domain)
form_ids = form_db.get_all_form_ids_in_domain()
form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived'))
for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)):
if isinstance(form, CouchForm):
meta = form.blobs.get(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
elif isinstance(form, XFormInstanceSQL):
meta = form.get_attachment_meta(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
else:
raise Exception("not sure how we got here")
def test_archie_modified_on(self):
case_id = uuid.uuid4().hex
now = datetime.utcnow().replace(microsecond=0)
earlier = now - timedelta(hours=1)
way_earlier = now - timedelta(days=1)
# make sure we timestamp everything so they have the right order
create_block = CaseBlock(case_id, create=True, date_modified=way_earlier)
post_case_blocks(
[create_block.as_xml()], form_extras={'received_on': way_earlier}
)
update_block = CaseBlock(case_id, update={'foo': 'bar'}, date_modified=earlier)
post_case_blocks(
[update_block.as_xml()], form_extras={'received_on': earlier}
)
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
self.assertEqual(earlier, case.modified_on)
second_form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(case.xform_ids[-1])
second_form.archive()
case = case_accessors.get_case(case_id)
self.assertEqual(way_earlier, case.modified_on)
def test_archive_only_form(self):
# check no data in stock states
ledger_accessors = LedgerAccessors(self.domain.name)
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(0, len(ledger_values))
initial_amounts = [(p._id, float(100)) for p in self.products]
form_id = self.submit_xml_form(balance_submission(initial_amounts))
# check that we made stuff
def _assert_initial_state():
if should_use_sql_backend(self.domain.name):
self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
self.assertEqual(100, int(state.stock_on_hand))
_assert_initial_state()
# archive and confirm commtrack data is cleared
form = FormAccessors(self.domain.name).get_form(form_id)
form.archive()
self.assertEqual(0, len(ledger_accessors.get_ledger_values_for_case(self.sp.case_id)))
if should_use_sql_backend(self.domain.name):
self.assertEqual(0, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(0, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=form_id).count())
# unarchive and confirm commtrack data is restored
form.unarchive()
_assert_initial_state()
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain, xmlns=None):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
self.xmlns = xmlns
def get_document(self, doc_id):
try:
form = self.form_accessors.get_form(doc_id)
if isinstance(form, XFormInstanceSQL):
return form.to_json(include_attachments=True)
else:
return form.to_json()
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns))
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield wrapped_form.to_json()
def setUp(self):
super(KafkaPublishingTest, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.form_accessors = FormAccessors(domain=self.domain)
self.processor = TestProcessor()
self.form_pillow = ConstructedPillow(
name='test-kafka-form-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'),
processor=self.processor
)
self.case_pillow = ConstructedPillow(
name='test-kafka-case-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'),
processor=self.processor
)
self.ledger_pillow = ConstructedPillow(
name='test-kafka-ledger-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'),
processor=self.processor
)
def reprocess_archive_stubs():
# Check for archive stubs
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from couchforms.models import UnfinishedArchiveStub
stubs = UnfinishedArchiveStub.objects.filter()
datadog_gauge('commcare.unfinished_archive_stubs', len(stubs))
start = time.time()
cutoff = start + timedelta(minutes=4).total_seconds()
for stub in stubs:
# Exit this task after 4 minutes so that the same stub isn't ever processed in multiple queues.
if time.time() - start > cutoff:
return
xform = FormAccessors(stub.domain).get_form(form_id=stub.xform_id)
# If the history wasn't updated the first time around, run the whole thing again.
if not stub.history_updated:
if stub.archive:
xform.archive(user_id=stub.user_id)
else:
xform.unarchive(user_id=stub.user_id)
# If the history was updated the first time around, just send the update to kafka
else:
xform.publish_archive_action_to_kafka(user_id=stub.user_id, archive=stub.archive)
def setUp(self):
super(BaseCaseMultimediaTest, self).setUp()
self.formdb = FormAccessors()
FormProcessorTestUtils.delete_all_cases()
FormProcessorTestUtils.delete_all_xforms()
def test_form_archiving(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True,
p1='p1-1',
p2='p2-1',
form_extras={'received_on': now})
_post_util(case_id=case_id,
p2='p2-2',
p3='p3-2',
p4='p4-2',
form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id,
p4='p4-3',
p5='p5-3',
close=True,
form_extras={'received_on': now + timedelta(seconds=2)})
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
closed_by = case.closed_by
closed_on = case.closed_on
self.assertNotEqual('', closed_by)
self.assertNotEqual(None, closed_on)
def _check_initial_state(case):
self.assertTrue(case.closed)
self.assertEqual(closed_by, case.closed_by)
self.assertEqual(closed_on, case.closed_on)
self.assertEqual(case.get_case_property('p1'), 'p1-1') # original
self.assertEqual(case.get_case_property('p2'),
'p2-2') # updated in second post
self.assertEqual(case.get_case_property('p3'),
'p3-2') # new in second post
self.assertEqual(case.get_case_property('p4'),
'p4-3') # updated in third post
self.assertEqual(case.get_case_property('p5'),
'p5-3') # new in third post
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
# SQL stores one transaction per form
self.assertEqual(3, len(
primary_actions(case))) # create + update + close
else:
self.assertEqual(5, len(
primary_actions(case))) # create + 3 updates + close
_check_initial_state(case)
# verify xform/action states
[f1, f2, f3] = case.xform_ids
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
[create, update, close] = case.actions
self.assertEqual(f1, create.form_id)
self.assertEqual(f2, update.form_id)
self.assertEqual(f3, close.form_id)
else:
[create, u1, u2, u3, close] = case.actions
self.assertEqual(f1, create.form_id)
self.assertEqual(f1, u1.form_id)
self.assertEqual(f2, u2.form_id)
self.assertEqual(f3, u3.form_id)
# todo: should this be the behavior for archiving the create form?
form_acessors = FormAccessors(REBUILD_TEST_DOMAIN)
f1_doc = form_acessors.get_form(f1)
with capture_kafka_changes_context(topics.CASE_SQL) as change_context:
f1_doc.archive()
if should_use_sql_backend(case.domain):
self.assertEqual([case.case_id],
[change.id for change in change_context.changes])
def __call__(self, item, context=None):
xforms_ids = CaseAccessors(item['domain']).get_case_xform_ids(
item['_id'])
forms = FormAccessors(item['domain']).get_forms(xforms_ids)
f_forms = [f for f in forms if f.xmlns == self.xmlns]
s_forms = sorted(f_forms, key=lambda x: x.received_on)
if len(s_forms) > 0:
latest_form = s_forms[-1]
else:
latest_form = None
path_to_action_plan = 'form/action_plan/%s/action_plan' % self.section
if latest_form:
action_plans = latest_form.get_data(path_to_action_plan)
if action_plans:
action_plan_for_question = None
for action_plan in action_plans:
if action_plan.get('incorrect_questions',
'') == self.question_id:
action_plan_for_question = action_plan
break
if action_plan_for_question:
incorrect_question = action_plan_for_question.get(
'incorrect_questions', '')
responsible = ', '.join([
item.get(x.strip(),
'---') for x in action_plan_for_question.get(
'action_plan_input', {}).get(
'responsible', '').split(',')
])
support = ', '.join([
item.get(x.strip(),
'---') for x in action_plan_for_question.get(
'action_plan_input', {}).get(
'support', '').split(',')
])
application = Application.get(latest_form.app_id)
form = application.get_form_by_xmlns(self.xmlns)
question_list = application.get_questions(self.xmlns)
questions = {x['value']: x for x in question_list}
return {
'form_name':
form.name['en'],
'section':
self.section,
'timeEnd':
latest_form.metadata.timeEnd,
'gap':
questions.get(
'data/code_to_text/%s' % incorrect_question,
{}).get('label', '---'),
'intervention_action':
action_plan_for_question.get('intervention_action',
'---'),
'responsible':
responsible,
'support':
support,
'deadline':
action_plan_for_question.get('DEADLINE', '---'),
'notes':
action_plan_for_question.get('notes', '---'),
}
]
if cases_to_delete:
with open(options['filename'], 'w') as csvfile:
writer = csv.writer(csvfile)
headers = [
'case id',
'case type',
'owner',
'opened by',
'app version',
]
writer.writerow(headers)
print(headers)
for case in cases_to_delete:
form = FormAccessors(domain=domain).get_form(
case.xform_ids[0])
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row = [
case.case_id,
case.type,
cached_owner_id_to_display(case.owner_id)
or case.owner_id,
cached_owner_id_to_display(case.opened_by),
app_version_info.build_version,
]
class CallCenterUtilsUserCaseTests(TestCase):
@classmethod
def setUpClass(cls):
super(CallCenterUtilsUserCaseTests, cls).setUpClass()
cls.domain = create_domain(TEST_DOMAIN)
cls.domain.usercase_enabled = True
cls.domain.save()
def setUp(self):
self.user = CommCareUser.create(TEST_DOMAIN, 'user1', '***', commit=False) # Don't commit yet
def tearDown(self):
self.user.delete()
@classmethod
def tearDownClass(cls):
delete_all_cases()
cls.domain.delete()
super(CallCenterUtilsUserCaseTests, cls).tearDownClass()
def test_sync_usercase_custom_user_data_on_create(self):
"""
Custom user data should be synced when the user is created
"""
self.user.user_data = {
'completed_training': 'yes',
}
self.user.save()
case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE)
self.assertIsNotNone(case)
self.assertEqual(case.dynamic_case_properties()['completed_training'], 'yes')
def test_sync_usercase_custom_user_data_on_update(self):
"""
Custom user data should be synced when the user is updated
"""
self.user.user_data = {
'completed_training': 'no',
}
self.user.save()
self.user.user_data = {
'completed_training': 'yes',
}
sync_usercase(self.user)
case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE)
self.assertEqual(case.dynamic_case_properties()['completed_training'], 'yes')
self._check_update_matches(case, {'completed_training': 'yes'})
def test_sync_usercase_overwrite_hq_props(self):
"""
Test that setting custom user data for owner_id and case_type don't change the case
"""
self.user.user_data = {
'owner_id': 'someone else',
'case_type': 'bob',
}
self.user.save()
case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE)
self.assertEqual(case.owner_id, self.user.get_id)
self.assertEqual(case.type, USERCASE_TYPE)
self.assertEqual(1, len(case.xform_ids))
def _check_update_matches(self, case, expected_update):
last_form = FormAccessors(TEST_DOMAIN).get_form(case.xform_ids[-1])
case_update = get_case_updates(last_form)[0]
self.assertDictEqual(case_update.update_block, expected_update)
if form.domain != domain:
return False
case_ids = get_case_ids_from_form(form)
# all cases touched by the form and not already modified
for case in CaseAccessors(domain).iter_cases(case_ids -
modified_cases):
if case.is_deleted != is_deletion:
# we can't delete/undelete this form - this would change the state of `case`
return False
# all cases touched by this form are deleted
return True
if is_deletion or Domain.get_by_name(domain).use_sql_backend:
all_forms = FormAccessors(domain).iter_forms(form_ids_to_modify)
else:
# accessor.iter_forms doesn't include deleted forms on the couch backend
all_forms = list(
map(FormAccessors(domain).get_form, form_ids_to_modify))
return [form.form_id for form in all_forms if _is_safe_to_modify(form)]
@task(serializer='pickle',
queue='background_queue',
ignore_result=True,
acks_late=True)
def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases,
deletion_id, deletion_date):
to_delete = _get_forms_to_modify(domain,
deleted_forms,
def test_archive_last_form(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
self.submit_xml_form(balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30))
final_amounts = [(p._id, float(50))
for i, p in enumerate(self.products)]
second_form_id = self.submit_xml_form(
balance_submission(final_amounts))
ledger_accessors = LedgerAccessors(self.domain.name)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(
3,
len(
self._get_all_ledger_transactions(
Q(form_id=second_form_id))))
else:
self.assertEqual(
1,
StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(
6,
StockTransaction.objects.filter(
report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(
self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(round(float(lv.daily_consumption), 2), 1.67)
# check initial setup
_assert_initial_state()
# archive and confirm commtrack data is deleted
form = FormAccessors(self.domain.name).get_form(second_form_id)
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.archive()
if should_use_sql_backend(self.domain):
self.assertEqual(
0,
len(
self._get_all_ledger_transactions(
Q(form_id=second_form_id))))
else:
self.assertEqual(
0,
StockReport.objects.filter(form_id=second_form_id).count())
self.assertEqual(
0,
StockTransaction.objects.filter(
report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(
self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
# balance should be reverted to 100 in the StockState
self.assertEqual(100, int(state.stock_on_hand))
# consumption should be none since there will only be 1 data point
self.assertIsNone(state.daily_consumption)
# unarchive and confirm commtrack data is restored
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.unarchive()
_assert_initial_state()
def payload_doc(self, repeat_record):
return FormAccessors(repeat_record.domain).get_form(repeat_record.payload_id)
def test_get_payload(self):
self.post_xml(self.xform_xml, self.domain)
payload_doc = FormAccessors(self.domain).get_form(self.instance_id)
payload = self.repeatergenerator.get_payload(None, payload_doc)
self.assertXmlEqual(self.xform_xml, payload)
def _get_forms_to_archive(self):
# ordered with latest form's id on top
form_accessor = FormAccessors(self.domain)
form_ids = form_accessor.get_form_ids_for_user(self.user_id)
return [f for f in form_accessor.get_forms(form_ids) if f.is_normal]
def setUp(self):
super(EditFormTest, self).setUp()
self.interface = FormProcessorInterface(self.domain)
self.casedb = CaseAccessors(self.domain)
self.formdb = FormAccessors(self.domain)
class EditFormTest(TestCase, TestFileMixin):
ID = '7H46J37FGH3'
domain = 'test-form-edits'
file_path = ('data', 'deprecation')
root = os.path.dirname(__file__)
def setUp(self):
super(EditFormTest, self).setUp()
self.interface = FormProcessorInterface(self.domain)
self.casedb = CaseAccessors(self.domain)
self.formdb = FormAccessors(self.domain)
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms(self.domain)
FormProcessorTestUtils.delete_all_cases(self.domain)
UnfinishedSubmissionStub.objects.all().delete()
super(EditFormTest, self).tearDown()
def test_basic_edit(self):
original_xml = self.get_xml('original')
edit_xml = self.get_xml('edit')
xform = submit_form_locally(original_xml, self.domain).xform
self.assertEqual(self.ID, xform.form_id)
self.assertTrue(xform.is_normal)
self.assertEqual("", xform.form_data['vitals']['height'])
self.assertEqual("other", xform.form_data['assessment']['categories'])
xform = submit_form_locally(edit_xml, self.domain).xform
self.assertEqual(self.ID, xform.form_id)
self.assertTrue(xform.is_normal)
self.assertEqual("100", xform.form_data['vitals']['height'])
self.assertEqual("Edited Baby!",
xform.form_data['assessment']['categories'])
self.assertEqual(1, len(xform.history))
self.assertEqual('edit', xform.history[0].operation)
deprecated_xform = self.formdb.get_form(xform.deprecated_form_id)
self.assertEqual(self.ID, deprecated_xform.orig_id)
self.assertNotEqual(self.ID, deprecated_xform.form_id)
self.assertTrue(deprecated_xform.is_deprecated)
self.assertEqual("", deprecated_xform.form_data['vitals']['height'])
self.assertEqual(
"other", deprecated_xform.form_data['assessment']['categories'])
self.assertEqual(xform.received_on, deprecated_xform.received_on)
self.assertEqual(xform.deprecated_form_id, deprecated_xform.form_id)
self.assertTrue(xform.edited_on > deprecated_xform.received_on)
self.assertEqual(deprecated_xform.get_xml().decode('utf-8'),
original_xml.decode('utf-8'))
self.assertEqual(xform.get_xml().decode('utf-8'),
edit_xml.decode('utf-8'))
def test_edit_form_with_attachments(self):
attachment_source = './corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg'
attachment_file = open(attachment_source, 'rb')
attachments = {
'fruity_file':
UploadedFile(attachment_file,
'fruity_file',
content_type='image/jpeg')
}
def _get_xml(date, form_id):
return """<?xml version='1.0' ?>
<data uiVersion="1" version="1" name="" xmlns="http://openrosa.org/formdesigner/123">
<name>fgg</name>
<date>2011-06-07</date>
<n1:meta xmlns:n1="http://openrosa.org/jr/xforms">
<n1:deviceID>354957031935664</n1:deviceID>
<n1:timeStart>{date}</n1:timeStart>
<n1:timeEnd>{date}</n1:timeEnd>
<n1:username>bcdemo</n1:username>
<n1:userID>user-abc</n1:userID>
<n1:instanceID>{form_id}</n1:instanceID>
</n1:meta>
</data>""".format(date=date,
attachment_source=attachment_source,
form_id=form_id)
form_id = uuid.uuid4().hex
original_xml = _get_xml('2016-03-01T12:04:16Z', form_id)
submit_form_locally(
original_xml,
self.domain,
attachments=attachments,
)
form = self.formdb.get_form(form_id)
self.assertIn('fruity_file', form.attachments)
self.assertIn(original_xml, form.get_xml().decode('utf-8'))
# edit form
edit_xml = _get_xml('2016-04-01T12:04:16Z', form_id)
submit_form_locally(
edit_xml,
self.domain,
)
form = self.formdb.get_form(form_id)
self.assertIsNotNone(form.edited_on)
self.assertIsNotNone(form.deprecated_form_id)
self.assertIn('fruity_file', form.attachments)
self.assertIn(edit_xml, form.get_xml().decode('utf-8'))
def test_edit_an_error(self):
form_id = uuid.uuid4().hex
case_block = CaseBlock.deprecated_init(
create=True,
case_id='', # this should cause the submission to error
case_type='person',
owner_id='some-owner',
)
form, _ = submit_case_blocks(case_block.as_text(),
domain=self.domain,
form_id=form_id)
self.assertTrue(form.is_error)
self.assertTrue('IllegalCaseId' in form.problem)
case_block.case_id = uuid.uuid4().hex
form, _ = submit_case_blocks(case_block.as_text(),
domain=self.domain,
form_id=form_id)
self.assertFalse(form.is_error)
self.assertEqual(None, getattr(form, 'problem', None))
def test_broken_save(self):
"""
Test that if the second form submission terminates unexpectedly
and the main form isn't saved, then there are no side effects
such as the original having been marked as deprecated.
"""
original_xml = self.get_xml('original')
edit_xml = self.get_xml('edit')
result = submit_form_locally(original_xml, self.domain)
xform = result.xform
self.assertEqual(self.ID, xform.form_id)
self.assertTrue(xform.is_normal)
self.assertEqual(self.domain, xform.domain)
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
0)
with patch.object(self.interface.processor,
'save_processed_models',
side_effect=HTTPError):
with self.assertRaises(HTTPError):
submit_form_locally(edit_xml, self.domain)
xform = self.formdb.get_form(self.ID)
self.assertIsNotNone(xform)
# it didn't go through, so make sure there are no edits still
self.assertIsNone(getattr(xform, 'deprecated_form_id', None))
self.assertEqual(
UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
0)
def test_case_management(self):
form_id = uuid.uuid4().hex
case_id = uuid.uuid4().hex
owner_id = uuid.uuid4().hex
case_block = CaseBlock.deprecated_init(create=True,
case_id=case_id,
case_type='person',
owner_id=owner_id,
update={
'property': 'original value'
}).as_text()
submit_case_blocks(case_block, domain=self.domain, form_id=form_id)
# validate some assumptions
case = self.casedb.get_case(case_id)
self.assertEqual(case.type, 'person')
self.assertEqual(case.dynamic_case_properties()['property'],
'original value')
self.assertEqual([form_id], case.xform_ids)
if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False):
self.assertEqual(2, len(case.actions))
for a in case.actions:
self.assertEqual(form_id, a.xform_id)
# submit a new form with a different case update
case_block = CaseBlock.deprecated_init(create=True,
case_id=case_id,
case_type='newtype',
owner_id=owner_id,
update={
'property': 'edited value'
}).as_text()
xform, _ = submit_case_blocks(case_block,
domain=self.domain,
form_id=form_id)
case = self.casedb.get_case(case_id)
self.assertEqual(case.type, 'newtype')
self.assertEqual(case.dynamic_case_properties()['property'],
'edited value')
self.assertEqual([form_id], case.xform_ids)
self.assertEqual(case.server_modified_on, xform.edited_on)
if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False):
self.assertEqual(2, len(case.actions))
for a in case.actions:
self.assertEqual(form_id, a.xform_id)
def test_second_edit_fails(self):
form_id = uuid.uuid4().hex
case_id = uuid.uuid4().hex
case_block = CaseBlock.deprecated_init(
create=True,
case_id=case_id,
case_type='person',
).as_text()
submit_case_blocks(case_block, domain=self.domain, form_id=form_id)
# submit an edit form with a bad case update (for example a bad ID)
case_block = CaseBlock.deprecated_init(
create=True,
case_id='',
case_type='person',
).as_text()
submit_case_blocks(case_block, domain=self.domain, form_id=form_id)
xform = self.formdb.get_form(form_id)
self.assertTrue(xform.is_error)
deprecated_xform = self.formdb.get_form(xform.deprecated_form_id)
self.assertTrue(deprecated_xform.is_deprecated)
def test_case_management_ordering(self):
case_id = uuid.uuid4().hex
owner_id = uuid.uuid4().hex
# create a case
case_block = CaseBlock.deprecated_init(
create=True,
case_id=case_id,
case_type='person',
owner_id=owner_id,
).as_text()
create_form_id = submit_case_blocks(case_block,
domain=self.domain)[0].form_id
# validate that worked
case = self.casedb.get_case(case_id)
self.assertEqual([create_form_id], case.xform_ids)
if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False):
self.assertTrue(
create_form_id in [a.xform_id for a in case.actions])
for a in case.actions:
self.assertEqual(create_form_id, a.xform_id)
edit_date = datetime.utcnow()
# set some property value
case_block = CaseBlock.deprecated_init(create=False,
case_id=case_id,
date_modified=edit_date,
update={
'property': 'first value',
}).as_text()
edit_form_id = submit_case_blocks(case_block,
domain=self.domain)[0].form_id
# validate that worked
case = self.casedb.get_case(case_id)
self.assertEqual(case.dynamic_case_properties()['property'],
'first value')
self.assertEqual([create_form_id, edit_form_id], case.xform_ids)
if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False):
self.assertTrue(
all(form_id in [a.xform_id for a in case.actions]
for form_id in [create_form_id, edit_form_id]))
# submit a second (new) form updating the value
case_block = CaseBlock.deprecated_init(create=False,
case_id=case_id,
update={
'property': 'final value',
}).as_text()
second_edit_form_id = submit_case_blocks(case_block,
domain=self.domain)[0].form_id
# validate that worked
case = self.casedb.get_case(case_id)
self.assertEqual(case.dynamic_case_properties()['property'],
'final value')
self.assertEqual([create_form_id, edit_form_id, second_edit_form_id],
case.xform_ids)
if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False):
self.assertTrue(
all(form_id in [a.xform_id for a in case.actions] for form_id
in [create_form_id, edit_form_id, second_edit_form_id]))
# deprecate the middle edit
case_block = CaseBlock.deprecated_init(
create=False,
case_id=case_id,
date_modified=
edit_date, # need to use the previous edit date for action sort comparisons
update={
'property': 'edited value',
'added_property': 'added value',
}).as_text()
submit_case_blocks(case_block,
domain=self.domain,
form_id=edit_form_id)
# ensure that the middle edit stays in the right place and is applied
# before the final one
case = self.casedb.get_case(case_id)
self.assertEqual(case.dynamic_case_properties()['property'],
'final value')
self.assertEqual(case.dynamic_case_properties()['added_property'],
'added value')
self.assertEqual([create_form_id, edit_form_id, second_edit_form_id],
case.xform_ids)
if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False):
self.assertEqual([
create_form_id, create_form_id, edit_form_id,
second_edit_form_id
], [a.xform_id for a in case.actions])
def test_edit_different_xmlns(self):
form_id = uuid.uuid4().hex
case1_id = uuid.uuid4().hex
case2_id = uuid.uuid4().hex
xmlns1 = 'http://commcarehq.org/xmlns1'
xmlns2 = 'http://commcarehq.org/xmlns2'
case_block = CaseBlock.deprecated_init(
create=True,
case_id=case1_id,
case_type='person',
owner_id='owner1',
).as_text()
xform, cases = submit_case_blocks(case_block,
domain=self.domain,
xmlns=xmlns1,
form_id=form_id)
self.assertTrue(xform.is_normal)
self.assertEqual(form_id, xform.form_id)
case_block = CaseBlock.deprecated_init(
create=True,
case_id=case2_id,
case_type='goat',
owner_id='owner1',
).as_text()
# submit new form with same form ID but different XMLNS
xform, cases = submit_case_blocks(case_block,
domain=self.domain,
xmlns=xmlns2,
form_id=form_id)
self.assertTrue(xform.is_normal)
self.assertNotEqual(form_id,
xform.form_id) # form should have a different ID
def test_copy_operations(self):
original_xml = self.get_xml('original')
edit_xml = self.get_xml('edit')
xform = submit_form_locally(original_xml, self.domain).xform
xform.archive(user_id='user1')
xform.unarchive(user_id='user2')
xform = submit_form_locally(edit_xml, self.domain).xform
self.assertEqual(3, len(xform.history))
self.assertEqual('archive', xform.history[0].operation)
self.assertEqual('unarchive', xform.history[1].operation)
self.assertEqual('edit', xform.history[2].operation)
class TestReprocessDuringSubmission(TestCase):
@classmethod
def setUpClass(cls):
super(TestReprocessDuringSubmission, cls).setUpClass()
cls.domain = uuid.uuid4().hex
def setUp(self):
super(TestReprocessDuringSubmission, self).setUp()
self.factory = CaseFactory(domain=self.domain)
self.formdb = FormAccessors(self.domain)
self.casedb = CaseAccessors(self.domain)
self.ledgerdb = LedgerAccessors(self.domain)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(TestReprocessDuringSubmission, self).tearDown()
def test_error_saving(self):
case_id = uuid.uuid4().hex
form_id = uuid.uuid4().hex
with _patch_save_to_raise_error(self):
submit_case_blocks(CaseBlock(case_id=case_id,
create=True,
case_type='box').as_text(),
self.domain,
form_id=form_id)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form = self.formdb.get_form(form_id)
self.assertTrue(form.is_error)
with self.assertRaises(CaseNotFound):
self.casedb.get_case(case_id)
result = submit_form_locally(
instance=form.get_xml(),
domain=self.domain,
)
duplicate_form = result.xform
self.assertTrue(duplicate_form.is_duplicate)
case = self.casedb.get_case(case_id)
self.assertIsNotNone(case)
form = self.formdb.get_form(form_id)
self.assertTrue(form.is_normal)
self.assertIsNone(getattr(form, 'problem', None))
self.assertEqual(duplicate_form.orig_id, form.form_id)
def test_processing_error(self):
case_id = uuid.uuid4().hex
parent_case_id = uuid.uuid4().hex
form_id = uuid.uuid4().hex
form, _ = submit_case_blocks(CaseBlock(
case_id=case_id,
create=True,
case_type='box',
index={
'cupboard': ('cupboard', parent_case_id)
},
).as_text(),
self.domain,
form_id=form_id)
self.assertTrue(form.is_error)
self.assertTrue('InvalidCaseIndex' in form.problem)
self.assertEqual(form.form_id, form_id)
with self.assertRaises(CaseNotFound):
self.casedb.get_case(case_id)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(0, len(stubs))
# create parent case
submit_case_blocks(
CaseBlock(case_id=parent_case_id,
create=True,
case_type='cupboard').as_text(),
self.domain,
)
# re-submit form
result = submit_form_locally(
instance=form.get_xml(),
domain=self.domain,
)
duplicate_form = result.xform
self.assertTrue(duplicate_form.is_duplicate)
case = self.casedb.get_case(case_id)
self.assertIsNotNone(case)
form = self.formdb.get_form(form_id)
self.assertTrue(form.is_normal)
self.assertIsNone(getattr(form, 'problem', None))
self.assertEqual(duplicate_form.orig_id, form.form_id)
def get_last_form_submission(self):
result = FormAccessors(self.domain).get_forms_by_type(
'XFormInstance', 1, recent_first=True)
return result[0] if len(result) > 0 else None
class SubmissionPost(object):
def __init__(self,
instance=None,
attachments=None,
auth_context=None,
domain=None,
app_id=None,
build_id=None,
path=None,
location=None,
submit_ip=None,
openrosa_headers=None,
last_sync_token=None,
received_on=None,
date_header=None,
partial_submission=False,
case_db=None):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.auth_context = self.auth_context.to_json()
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
instance = _transform_instance_to_error(self.interface, error,
instance)
xforms[0] = instance
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old,
'Form with old metadata submitted',
xform.form_id)
def run(self):
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [])
result = process_xform_xml(self.domain, self.instance,
self.attachments)
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form.user_id)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = self.get_exception_response_and_log(
submitted_form, self.path)
return FormProcessingResult(response, None, [], [])
cases = []
ledgers = []
with result.get_locked_forms() as xforms:
from casexml.apps.case.xform import get_and_check_xform_domain
domain = get_and_check_xform_domain(xforms[0])
if self.case_db:
assert self.case_db.domain == domain
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(domain=domain,
lock=True,
deleted_ok=True,
xforms=xforms)
with case_db_cache as case_db:
instance = xforms[0]
if instance.xmlns == DEVICE_LOG_XMLNS:
try:
process_device_log(self.domain, instance)
except Exception:
notify_exception(None,
"Error processing device log",
details={
'xml': self.instance,
'domain': self.domain
})
raise
elif instance.is_duplicate:
self.interface.save_processed_models([instance])
elif not instance.is_error:
try:
case_stock_result = self.process_xforms_for_cases(
xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex) as e:
self._handle_known_error(e, instance, xforms)
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
self.save_processed_models(xforms, case_stock_result)
case_stock_result.case_result.close_extensions(case_db)
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
errors = self.process_signals(instance)
response = self._get_open_rosa_response(instance, errors)
return FormProcessingResult(response, instance, cases, ledgers)
@property
def _cache(self):
return get_redis_default_cache()
@property
def _restore_cache_key(self):
from casexml.apps.phone.restore import restore_cache_key
return restore_cache_key
def _invalidate_caches(self, user_id):
"""invalidate cached initial restores"""
initial_restore_cache_key = self._restore_cache_key(
RESTORE_CACHE_KEY_PREFIX, user_id, version=V2)
self._cache.delete(initial_restore_cache_key)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_caches(user_id)
def _invalidate_async_caches(self, user_id):
cache_key = self._restore_cache_key(ASYNC_RESTORE_CACHE_KEY_PREFIX,
user_id)
task_id = self._cache.get(cache_key)
if task_id is not None:
revoke_celery_task(task_id)
self._cache.delete(cache_key)
def save_processed_models(self, xforms, case_stock_result):
from casexml.apps.case.signals import case_post_save
instance = xforms[0]
with unfinished_submission(instance) as unfinished_submission_stub:
self.interface.save_processed_models(
xforms, case_stock_result.case_models,
case_stock_result.stock_result)
unfinished_submission_stub.saved = True
unfinished_submission_stub.save()
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
for case in case_stock_result.case_models:
case_post_save.send(case.__class__, case=case)
def process_xforms_for_cases(self, xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
cases = case_db.get_cases_for_saving(instance.received_on)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
def process_signals(self, instance):
feedback = successful_form_received.send_robust(None, xform=instance)
errors = []
for func, resp in feedback:
if resp and isinstance(resp, Exception):
error_message = unicode(resp)
logging.error((u"Receiver app: problem sending "
u"post-save signal %s for xform %s: %s: %s") %
(func, instance.form_id, type(resp).__name__,
error_message))
errors.append(error_message)
if errors:
self.interface.xformerror_from_xform_instance(
instance, ", ".join(errors))
self.formdb.update_form_problem_and_state(instance)
return errors
def _get_open_rosa_response(self, instance, errors):
if instance.is_normal:
response = self.get_success_response(instance, errors)
else:
response = self.get_failure_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_success_response(doc, errors):
if errors:
response = OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
else:
response = OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u' √ ',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
return response
@staticmethod
def submission_ignored_response():
return OpenRosaResponse(
# would have done ✓ but our test Nokias' fonts don't have that character
message=u'√ (this submission was ignored)',
nature=ResponseNature.SUBMIT_SUCCESS,
status=201,
).response()
@staticmethod
def get_failure_response(doc):
return OpenRosaResponse(
message=doc.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=201,
).response()
@staticmethod
def get_exception_response_and_log(error_instance, path):
logging.exception(
u"Problem receiving submission to %s. Doc id: %s, Error %s" %
(path, error_instance.form_id, error_instance.problem))
return OpenRosaResponse(
message="There was an error processing the form: %s" %
error_instance.problem,
nature=ResponseNature.SUBMIT_ERROR,
status=500,
).response()
@staticmethod
def get_blacklisted_response():
return OpenRosaResponse(
message=(
"This submission was blocked because of an unusual volume "
"of submissions from this project space. Please contact "
"support to resolve."),
nature=ResponseNature.SUBMIT_ERROR,
status=509,
).response()
session_name = u'{app} > {form}'.format(
app=app.name,
form=form_name,
)
if case_id:
case = CaseAccessors(domain).get_case(case_id)
session_name = u'{0} - {1}'.format(session_name, case.name)
root_context = {
'form_url': form_url,
'formplayer_url': settings.FORMPLAYER_URL,
}
if instance_id:
try:
root_context['instance_xml'] = FormAccessors(domain).get_form(
instance_id).get_xml()
except XFormNotFound:
raise Http404()
session_extras = {'session_name': session_name, 'app_id': app._id}
session_extras.update(
get_cloudcare_session_data(domain, form, request.couch_user))
delegation = request.GET.get('task-list') == 'true'
session_helper = CaseSessionDataHelper(domain,
request.couch_user,
case_id,
app,
form,
delegation=delegation)
return json_response(
def handle_unexpected_error(interface, instance, exception):
instance = _transform_instance_to_error(interface, exception, instance)
notify_submission_error(instance, exception, instance.problem)
FormAccessors(interface.domain).save_new_form(instance)
def setUp(self):
super(FundamentalBaseTests, self).setUp()
self.interface = FormProcessorInterface()
self.casedb = CaseAccessors(DOMAIN)
self.formdb = FormAccessors(DOMAIN)
class Command(BaseCommand):
help = "Delete all cases that are in a specific case's network/footprint"
def add_arguments(self, parser):
parser.add_argument('domain', type=unicode)
parser.add_argument('case_id', type=unicode)
parser.add_argument('--filename',
dest='filename',
default='case-delete-info.csv')
def handle(self, *args, **options):
domain = options['domain']
case_id = options['case_id']
case_accessor = CaseAccessors(domain=domain)
case = case_accessor.get_case(case_id)
if not case.is_deleted and raw_input('\n'.join([
'Case {} is not already deleted. Are you sure you want to delete it? (y/N)'
.format(case_id)
])).lower() != 'y':
sys.exit(0)
dependent_case_ids = get_entire_case_network(domain, [case_id])
cases_to_delete = filter(lambda case: not case.is_deleted,
case_accessor.get_cases(dependent_case_ids))
if cases_to_delete:
with open(options['filename'], 'w') as csvfile:
writer = csv.writer(csvfile)
headers = [
'case id',
'case type',
'owner',
'opened by',
'app version',
]
writer.writerow(headers)
print headers
for case in cases_to_delete:
form = FormAccessors(domain=domain).get_form(
case.xform_ids[0])
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row = [
case.case_id,
case.type,
cached_owner_id_to_display(case.owner_id)
or case.owner_id,
cached_owner_id_to_display(case.opened_by),
app_version_info.build_version,
]
writer.writerow(row)
print row
if cases_to_delete and raw_input('\n'.join([
'Delete these {} cases? (y/N)'.format(len(cases_to_delete)),
])).lower() == 'y':
case_accessor.soft_delete_cases(
[c.case_id for c in cases_to_delete])
print 'deleted {} cases'.format(len(cases_to_delete))
if cases_to_delete:
print 'details here: {}'.format(options['filename'])
else:
print "didn't find any cases to delete"
def get_forms(self):
from corehq.apps.api.util import form_to_es_form
forms = FormAccessors(self.domain).get_forms(self.xform_ids)
return list(filter(None, [form_to_es_form(form) for form in forms]))
class TestFormArchiving(TestCase, TestFileMixin):
file_path = ('data', 'sample_xforms')
root = os.path.dirname(__file__)
def setUp(self):
super(TestFormArchiving, self).setUp()
self.casedb = CaseAccessors('test-domain')
self.formdb = FormAccessors('test-domain')
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms()
FormProcessorTestUtils.delete_all_cases()
super(TestFormArchiving, self).tearDown()
def testArchive(self):
case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169'
xml_data = self.get_xml('basic')
result = submit_form_locally(
xml_data,
'test-domain',
)
xform = result.xform
self.assertTrue(xform.is_normal)
self.assertEqual(0, len(xform.history))
lower_bound = datetime.utcnow() - timedelta(seconds=1)
xform.archive(user_id='mr. librarian')
upper_bound = datetime.utcnow() + timedelta(seconds=1)
xform = self.formdb.get_form(xform.form_id)
self.assertTrue(xform.is_archived)
case = self.casedb.get_case(case_id)
self.assertTrue(case.is_deleted)
self.assertEqual(case.xform_ids, [])
[archival] = xform.history
self.assertTrue(lower_bound <= archival.date <= upper_bound)
self.assertEqual('archive', archival.operation)
self.assertEqual('mr. librarian', archival.user)
lower_bound = datetime.utcnow() - timedelta(seconds=1)
xform.unarchive(user_id='mr. researcher')
upper_bound = datetime.utcnow() + timedelta(seconds=1)
xform = self.formdb.get_form(xform.form_id)
self.assertTrue(xform.is_normal)
case = self.casedb.get_case(case_id)
self.assertFalse(case.is_deleted)
self.assertEqual(case.xform_ids, [xform.form_id])
[archival, restoration] = xform.history
self.assertTrue(lower_bound <= restoration.date <= upper_bound)
self.assertEqual('unarchive', restoration.operation)
self.assertEqual('mr. researcher', restoration.user)
def testSignal(self):
global archive_counter, restore_counter
archive_counter = 0
restore_counter = 0
def count_archive(**kwargs):
global archive_counter
archive_counter += 1
def count_unarchive(**kwargs):
global restore_counter
restore_counter += 1
xform_archived.connect(count_archive)
xform_unarchived.connect(count_unarchive)
xml_data = self.get_xml('basic')
result = submit_form_locally(
xml_data,
'test-domain',
)
self.assertEqual(0, archive_counter)
self.assertEqual(0, restore_counter)
result.xform.archive()
self.assertEqual(1, archive_counter)
self.assertEqual(0, restore_counter)
xform = self.formdb.get_form(result.xform.form_id)
xform.unarchive()
self.assertEqual(1, archive_counter)
self.assertEqual(1, restore_counter)
def inaccessible_forms_accessed(self, xform_ids, domain, couch_user):
xforms = FormAccessors(domain).get_forms(xform_ids)
xforms_user_ids = set([xform.user_id for xform in xforms])
accessible_user_ids = set(user_ids_at_accessible_locations(domain, couch_user))
return xforms_user_ids - accessible_user_ids
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer, app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain, xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
class ReprocessSubmissionStubTests(TestCase):
@classmethod
def setUpClass(cls):
super(ReprocessSubmissionStubTests, cls).setUpClass()
cls.domain = uuid.uuid4().hex
cls.product = SQLProduct.objects.create(domain=cls.domain,
product_id='product1',
name='product1')
@classmethod
def tearDownClass(cls):
cls.product.delete()
super(ReprocessSubmissionStubTests, cls).tearDownClass()
def setUp(self):
super(ReprocessSubmissionStubTests, self).setUp()
self.factory = CaseFactory(domain=self.domain)
self.formdb = FormAccessors(self.domain)
self.casedb = CaseAccessors(self.domain)
self.ledgerdb = LedgerAccessors(self.domain)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(ReprocessSubmissionStubTests, self).tearDown()
def test_reprocess_unfinished_submission_case_create(self):
case_id = uuid.uuid4().hex
with _patch_save_to_raise_error(self):
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id,
attrs={
'case_type': 'parent',
'create': True
})
])
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
# form that was saved before case error raised
normal_form_ids = self.formdb.get_all_form_ids_in_domain(
'XFormInstance')
self.assertEqual(0, len(normal_form_ids))
# shows error form (duplicate of form that was saved before case error)
# this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed
# the form didn't get saved
# we don't really care about this form in this test
error_forms = self.formdb.get_forms_by_type('XFormError', 10)
self.assertEqual(1, len(error_forms))
self.assertIsNone(error_forms[0].orig_id)
self.assertEqual(error_forms[0].form_id, stubs[0].xform_id)
self.assertEqual(0, len(self.casedb.get_case_ids_in_domain()))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
case_ids = self.casedb.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
self.assertEqual(case_id, case_ids[0])
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_reprocess_unfinished_submission_case_update(self):
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(
submit_case_blocks(
CaseBlock(case_id=case_id, create=True,
case_type='box').as_text(), self.domain)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
CaseBlock(case_id=case_id, update={
'prop': 'a'
}).as_text(), self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit second form with case update
form_ids.append(
submit_case_blocks(
CaseBlock(case_id=case_id, update={
'prop': 'b'
}).as_text(), self.domain)[0].form_id)
case = self.casedb.get_case(case_id)
self.assertEqual(2, len(case.xform_ids))
self.assertEqual('b', case.get_case_property('prop'))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(0, len(result.ledgers))
case = self.casedb.get_case(case_id)
self.assertEqual('b', case.get_case_property(
'prop')) # should be property value from most recent form
self.assertEqual(3, len(case.xform_ids))
self.assertEqual(form_ids, case.xform_ids)
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_reprocess_unfinished_submission_ledger_create(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id,
attrs={
'case_type': 'parent',
'create': True
})
])
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 100),
self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(0, len(ledgers))
case = self.casedb.get_case(case_id)
self.assertEqual(1, len(case.xform_ids))
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
self.assertEqual(0, len(ledger_transactions))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
self.assertEqual(1, len(ledger_transactions))
# case still only has 2 transactions
case = self.casedb.get_case(case_id)
self.assertEqual(2, len(case.xform_ids))
if should_use_sql_backend(self.domain):
self.assertTrue(case.actions[1].is_ledger_transaction)
def test_reprocess_unfinished_submission_ledger_rebuild(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(
submit_case_blocks([
CaseBlock(case_id=case_id, create=True,
case_type='shop').as_text(),
get_single_balance_block(case_id, 'product1', 100),
], self.domain)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 50), self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit another form afterwards
form_ids.append(
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 25),
self.domain)[0].form_id)
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(2, len(ledger_transactions))
else:
# includes extra consumption transaction
self.assertEqual(3, len(ledger_transactions))
# should rebuild ledger transactions
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers)) # still only 1
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(ledger_transactions))
# make sure transactions are in correct order
self.assertEqual(form_ids,
[trans.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].updated_balance)
self.assertEqual(100, ledger_transactions[0].delta)
self.assertEqual(50, ledger_transactions[1].updated_balance)
self.assertEqual(-50, ledger_transactions[1].delta)
self.assertEqual(25, ledger_transactions[2].updated_balance)
self.assertEqual(-25, ledger_transactions[2].delta)
else:
self.assertEqual(3, len(ledger_transactions))
self.assertEqual(
form_ids,
[trans.report.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].stock_on_hand)
self.assertEqual(50, ledger_transactions[1].stock_on_hand)
self.assertEqual(25, ledger_transactions[2].stock_on_hand)
def test_fire_signals(self):
from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler
case_id = uuid.uuid4().hex
form_id = uuid.uuid4().hex
with failing_signal_handler('signal death'):
submit_case_blocks(CaseBlock(case_id=case_id,
create=True,
case_type='box').as_text(),
self.domain,
form_id=form_id)
form = self.formdb.get_form(form_id)
with catch_signal(
successful_form_received) as form_handler, catch_signal(
case_post_save) as case_handler:
submit_form_locally(
instance=form.get_xml(),
domain=self.domain,
)
case = self.casedb.get_case(case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form, form_handler.call_args[1]['xform'])
self.assertEqual(case, case_handler.call_args[1]['case'])
else:
signal_form = form_handler.call_args[1]['xform']
self.assertEqual(form.form_id, signal_form.form_id)
self.assertEqual(form.get_rev, signal_form.get_rev)
signal_case = case_handler.call_args[1]['case']
self.assertEqual(case.case_id, signal_case.case_id)
self.assertEqual(case.get_rev, signal_case.get_rev)
class BaseCaseMultimediaTest(TestCase, TestFileMixin):
file_path = ('data', 'multimedia')
root = os.path.dirname(__file__)
def setUp(self):
super(BaseCaseMultimediaTest, self).setUp()
self.formdb = FormAccessors()
FormProcessorTestUtils.delete_all_cases()
FormProcessorTestUtils.delete_all_xforms()
def _formatXForm(self, doc_id, raw_xml, attachment_block, date=None):
if date is None:
date = datetime.utcnow()
final_xml = Template(raw_xml.decode("utf8")).render(
Context({
"attachments":
attachment_block,
"time_start":
json_format_datetime(date - timedelta(minutes=4)),
"time_end":
json_format_datetime(date),
"date_modified":
json_format_datetime(date),
"doc_id":
doc_id
})).encode("utf8")
return final_xml
def _prepAttachments(self, new_attachments, removes=[]):
"""
Returns:
attachment_block - An XML representation of the attachment
dict_attachments - A key-value dict where the key is the name and the value is a Stream of the
attachment
"""
attachment_block = ''.join(
[self._singleAttachBlock(x) for x in new_attachments] +
[self._singleAttachRemoveBlock(x) for x in removes])
dict_attachments = dict(
(MEDIA_FILES[attach_name], self._attachmentFileStream(attach_name))
for attach_name in new_attachments)
return attachment_block, dict_attachments
def _singleAttachBlock(self, key):
return '<n0:%s src="%s" from="local"/>' % (key, MEDIA_FILES[key])
def _singleAttachRemoveBlock(self, key):
return '<n0:%s />' % key
def _attachmentFileStream(self, key):
attachment_path = MEDIA_FILES[key]
attachment = open(attachment_path, 'rb')
return NoClose(UploadedFile(attachment, key))
def _calc_file_hash(self, key):
with open(MEDIA_FILES[key], 'rb') as attach:
return hashlib.md5(attach.read()).hexdigest()
def _do_submit(self,
xml_data,
dict_attachments,
sync_token=None,
date=None):
"""
RequestFactory submitter - simulates direct submission to server
directly (no need to call process case after fact)
"""
with flag_enabled('MM_CASE_PROPERTIES'):
result = submit_form_locally(xml_data,
TEST_DOMAIN_NAME,
attachments=dict_attachments,
last_sync_token=sync_token,
received_on=date)
xform = result.xform
attachments = xform.attachments
self.assertEqual(set(dict_attachments.keys()), set(attachments.keys()))
self.assertEqual(result.case.case_id, TEST_CASE_ID)
return result.response, self.formdb.get_form(
xform.form_id), result.cases
def _submit_and_verify(self,
doc_id,
xml_data,
dict_attachments,
sync_token=None,
date=None):
response, form, [case] = self._do_submit(xml_data,
dict_attachments,
sync_token,
date=date)
attachments = form.attachments
self.assertEqual(len(dict_attachments), len(attachments))
for k, vstream in dict_attachments.items():
fileback = form.get_attachment(k)
# rewind the pointer before comparing
orig_attachment = vstream
orig_attachment.seek(0)
self.assertEqual(
hashlib.md5(fileback).hexdigest(),
hashlib.md5(orig_attachment.read()).hexdigest())
case = CaseAccessors(TEST_DOMAIN_NAME).get_case(
case.case_id) # re-fetch case
return form, case
def _doCreateCaseWithMultimedia(self, attachments=['fruity_file']):
xml_data = self.get_xml('multimedia_create')
attachment_block, dict_attachments = self._prepAttachments(attachments)
final_xml = self._formatXForm(CREATE_XFORM_ID, xml_data,
attachment_block)
return self._submit_and_verify(CREATE_XFORM_ID, final_xml,
dict_attachments)
def _doSubmitUpdateWithMultimedia(self,
new_attachments=None,
removes=None,
sync_token=None,
date=None):
new_attachments = new_attachments if new_attachments is not None \
else ['commcare_logo_file', 'dimagi_logo_file']
removes = removes if removes is not None else ['fruity_file']
attachment_block, dict_attachments = self._prepAttachments(
new_attachments, removes=removes)
raw_xform = self.get_xml('multimedia_update')
doc_id = uuid.uuid4().hex
final_xform = self._formatXForm(doc_id, raw_xform, attachment_block,
date)
return self._submit_and_verify(doc_id,
final_xform,
dict_attachments,
sync_token,
date=date)
def setUp(self):
super(TestFormArchiving, self).setUp()
self.casedb = CaseAccessors('test-domain')
self.formdb = FormAccessors('test-domain')
def archive_or_restore_forms(domain,
user_id,
username,
form_ids,
archive_or_restore,
task=None,
from_excel=False):
response = {
'errors': [],
'success': [],
}
missing_forms = set(form_ids)
success_count = 0
if task:
DownloadBase.set_progress(task, 0, len(form_ids))
for xform in FormAccessors(domain).iter_forms(form_ids):
missing_forms.discard(xform.form_id)
if xform.domain != domain:
response['errors'].append(
_("XForm {form_id} does not belong to domain {domain}").format(
form_id=xform.form_id, domain=domain))
continue
xform_string = _(
"XForm {form_id} for domain {domain} by user '{username}'").format(
form_id=xform.form_id, domain=xform.domain, username=username)
try:
if archive_or_restore.is_archive_mode():
xform.archive(user_id=user_id)
message = _("Successfully archived {form}").format(
form=xform_string)
else:
xform.unarchive(user_id=user_id)
message = _("Successfully unarchived {form}").format(
form=xform_string)
response['success'].append(message)
success_count = success_count + 1
except Exception as e:
response['errors'].append(
_("Could not archive {form}: {error}").format(
form=xform_string, error=e))
if task:
DownloadBase.set_progress(task, success_count, len(form_ids))
for missing_form_id in missing_forms:
response['errors'].append(
_("Could not find XForm {form_id}").format(
form_id=missing_form_id))
if from_excel:
return response
response["success_count_msg"] = _("{success_msg} {count} form(s)".format(
success_msg=archive_or_restore.success_text, count=success_count))
return {"messages": response}
def _get_form_attachment_info(domain, form_ids, export):
properties = _get_export_properties(export)
return [
_extract_form_attachment_info(form, properties)
for form in FormAccessors(domain).iter_forms(form_ids)
]
def handle(self, **options):
domain = options['domain']
debug = options['debug']
cleanup = options['cleanup']
domain_query = CaseES().domain(domain)
valid_case_ids = set(domain_query.get_ids())
referenced_case_ids = {
index['referenced_id']
for hit in domain_query.source('indices.referenced_id').run().hits
for index in hit['indices']
}
invalid_referenced_ids = referenced_case_ids - valid_case_ids
if len(invalid_referenced_ids) > ES_MAX_CLAUSE_COUNT:
print("there's a lot of invalid ids here. ES queries may not handle this well")
cases_with_invalid_references = (
domain_query
.term('indices.referenced_id', invalid_referenced_ids)
.source(['_id', 'type', 'indices', 'owner_id', 'opened_by', 'xform_ids'])
.run().hits
)
with open(options['filename'], 'w', encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
headers = [
'case id',
'case type',
'creating form id',
'referenced id',
'referenced_type',
'index relationship',
'index identifier',
'owner id',
'owner name',
'opened by id',
'opened by name',
]
if debug:
headers.append('app version')
writer.writerow(headers)
for case in cases_with_invalid_references:
for index in case['indices']:
if index['referenced_id'] in invalid_referenced_ids:
form_id = case['xform_ids'][0]
row = [
case['_id'],
case['type'],
form_id,
index['referenced_id'],
index['referenced_type'],
index['relationship'],
index['identifier'],
case['owner_id'],
cached_owner_id_to_display(case['owner_id']),
case['opened_by'],
cached_owner_id_to_display(case['opened_by']),
]
if debug:
form = FormAccessors(domain=domain).get_form(form_id)
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row.append(app_version_info.build_version)
writer.writerow(row)
if cleanup:
missing = set()
deleted = set()
exists = set()
for invalid_id in invalid_referenced_ids:
try:
case = CaseAccessors(domain).get_case(invalid_id)
except CaseNotFound:
missing.add(invalid_id)
else:
if case.is_deleted:
deleted.add(case)
else:
exists.add(case)
class SubmissionPost(object):
def __init__(self, instance=None, attachments=None, auth_context=None,
domain=None, app_id=None, build_id=None, path=None,
location=None, submit_ip=None, openrosa_headers=None,
last_sync_token=None, received_on=None, date_header=None,
partial_submission=False, case_db=None, force_logs=False):
assert domain, "'domain' is required"
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
if case_db:
assert case_db.domain == domain
self.force_logs = force_logs
self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3
self.track_load = form_load_counter("form_submission", domain)
def _set_submission_properties(self, xform):
# attaches shared properties of the request to the document.
# used on forms and errors
xform.submit_ip = self.submit_ip
xform.path = self.path
xform.openrosa_headers = self.openrosa_headers
xform.last_sync_token = self.last_sync_token
if self.received_on:
xform.received_on = self.received_on
if self.date_header:
xform.date_header = self.date_header
xform.app_id = self.app_id
xform.build_id = self.build_id
xform.export_tag = ["domain", "xmlns"]
xform.partial_submission = self.partial_submission
return xform
def _handle_known_error(self, error, instance, xforms):
# errors we know about related to the content of the form
# log the error and respond with a success code so that the phone doesn't
# keep trying to send the form
xforms[0] = _transform_instance_to_error(self.interface, error, instance)
# this is usually just one document, but if an edit errored we want
# to save the deprecated form as well
self.interface.save_processed_models(xforms)
def _handle_basic_failure_modes(self):
if any_migrations_in_progress(self.domain):
# keep submissions on the phone
# until ready to start accepting again
return HttpResponse(status=503)
if not self.auth_context.is_valid():
return HttpResponseForbidden('Bad auth')
if isinstance(self.instance, BadRequest):
return HttpResponseBadRequest(self.instance.message)
def _post_process_form(self, xform):
self._set_submission_properties(xform)
found_old = scrub_meta(xform)
legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id)
def _get_success_message(self, instance, cases=None):
'''
Formplayer requests get a detailed success message pointing to the form/case affected.
All other requests get a generic message.
Message is formatted with markdown.
'''
if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID:
return ' √ '
messages = []
user = CouchUser.get_by_user_id(instance.user_id)
if not user or not user.is_web_user():
return _('Form successfully saved!')
from corehq.apps.export.views.list import CaseExportListView, FormExportListView
from corehq.apps.export.views.utils import can_view_case_exports, can_view_form_exports
from corehq.apps.reports.views import CaseDataView, FormDataView
form_link = case_link = form_export_link = case_export_link = None
form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory'
if has_permission_to_view_report(user, instance.domain, form_view):
form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id])
case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport'
if cases and has_permission_to_view_report(user, instance.domain, case_view):
if len(cases) == 1:
case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id])
else:
case_link = ", ".join(["[{}]({})".format(
c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id])
) for c in cases])
if can_view_form_exports(user, instance.domain):
form_export_link = reverse(FormExportListView.urlname, args=[instance.domain])
if cases and can_view_case_exports(user, instance.domain):
case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain])
# Start with generic message
messages.append(_('Form successfully saved!'))
# Add link to form/case if possible
if form_link and case_link:
if len(cases) == 1:
messages.append(
_("You submitted [this form]({}), which affected [this case]({}).")
.format(form_link, case_link))
else:
messages.append(
_("You submitted [this form]({}), which affected these cases: {}.")
.format(form_link, case_link))
elif form_link:
messages.append(_("You submitted [this form]({}).").format(form_link))
elif case_link:
if len(cases) == 1:
messages.append(_("Your form affected [this case]({}).").format(case_link))
else:
messages.append(_("Your form affected these cases: {}.").format(case_link))
# Add link to all form/case exports
if form_export_link and case_export_link:
messages.append(
_("Click to export your [case]({}) or [form]({}) data.")
.format(case_export_link, form_export_link))
elif form_export_link:
messages.append(_("Click to export your [form data]({}).").format(form_export_link))
elif case_export_link:
messages.append(_("Click to export your [case data]({}).").format(case_export_link))
return "\n\n".join(messages)
def run(self):
self.track_load()
failure_response = self._handle_basic_failure_modes()
if failure_response:
return FormProcessingResult(failure_response, None, [], [], 'known_failures')
result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json())
submitted_form = result.submitted_form
self._post_process_form(submitted_form)
self._invalidate_caches(submitted_form)
if submitted_form.is_submission_error_log:
self.formdb.save_new_form(submitted_form)
response = None
try:
xml = self.instance.decode()
except UnicodeDecodeError:
pass
else:
if 'log_subreport' in xml:
response = self.get_exception_response_and_log(
'Badly formed device log', submitted_form, self.path
)
if not response:
response = self.get_exception_response_and_log(
'Problem receiving submission', submitted_form, self.path
)
return FormProcessingResult(response, None, [], [], 'submission_error_log')
if submitted_form.xmlns == SYSTEM_ACTION_XMLNS:
return self.handle_system_action(submitted_form)
if submitted_form.xmlns == DEVICE_LOG_XMLNS:
return self.process_device_log(submitted_form)
cases = []
ledgers = []
submission_type = 'unknown'
openrosa_kwargs = {}
with result.get_locked_forms() as xforms:
if len(xforms) > 1:
self.track_load(len(xforms) - 1)
if self.case_db:
case_db_cache = self.case_db
case_db_cache.cached_xforms.extend(xforms)
else:
case_db_cache = self.interface.casedb_cache(
domain=self.domain, lock=True, deleted_ok=True,
xforms=xforms, load_src="form_submission",
)
with case_db_cache as case_db:
instance = xforms[0]
if instance.is_duplicate:
with tracer.trace('submission.process_duplicate'):
submission_type = 'duplicate'
existing_form = xforms[1]
stub = UnfinishedSubmissionStub.objects.filter(
domain=instance.domain,
xform_id=existing_form.form_id
).first()
result = None
if stub:
from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form
result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False)
elif existing_form.is_error:
from corehq.form_processor.reprocess import reprocess_form
result = reprocess_form(existing_form, lock_form=False)
if result and result.error:
submission_type = 'error'
openrosa_kwargs['error_message'] = result.error
if existing_form.is_error:
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
else:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
else:
self.interface.save_processed_models([instance])
elif not instance.is_error:
submission_type = 'normal'
try:
case_stock_result = self.process_xforms_for_cases(xforms, case_db)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
self._handle_known_error(e, instance, xforms)
submission_type = 'error'
openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE
except Exception as e:
# handle / log the error and reraise so the phone knows to resubmit
# note that in the case of edit submissions this won't flag the previous
# submission as having been edited. this is intentional, since we should treat
# this use case as if the edit "failed"
handle_unexpected_error(self.interface, instance, e)
raise
else:
instance.initial_processing_complete = True
openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
case_stock_result)
if openrosa_kwargs['error_message']:
openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE
cases = case_stock_result.case_models
ledgers = case_stock_result.stock_result.models_to_save
openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases)
elif instance.is_error:
submission_type = 'error'
response = self._get_open_rosa_response(instance, **openrosa_kwargs)
return FormProcessingResult(response, instance, cases, ledgers, submission_type)
def _conditionally_send_device_logs_to_sumologic(self, instance):
url = getattr(settings, 'SUMOLOGIC_URL', None)
if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER):
SumoLogicLog(self.domain, instance).send_data(url)
def _invalidate_caches(self, xform):
for device_id in {None, xform.metadata.deviceID if xform.metadata else None}:
self._invalidate_restore_payload_path_cache(xform, device_id)
if ASYNC_RESTORE.enabled(self.domain):
self._invalidate_async_restore_task_id_cache(xform, device_id)
def _invalidate_restore_payload_path_cache(self, xform, device_id):
"""invalidate cached initial restores"""
restore_payload_path_cache = RestorePayloadPathCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=xform.last_sync_token,
device_id=device_id,
)
restore_payload_path_cache.invalidate()
def _invalidate_async_restore_task_id_cache(self, xform, device_id):
async_restore_task_id_cache = AsyncRestoreTaskIdCache(
domain=self.domain,
user_id=xform.user_id,
sync_log_id=self.last_sync_token,
device_id=device_id,
)
task_id = async_restore_task_id_cache.get_value()
if task_id is not None:
revoke_celery_task(task_id)
async_restore_task_id_cache.invalidate()
@tracer.wrap(name='submission.save_models')
def save_processed_models(self, case_db, xforms, case_stock_result):
instance = xforms[0]
try:
with unfinished_submission(instance) as unfinished_submission_stub:
try:
self.interface.save_processed_models(
xforms,
case_stock_result.case_models,
case_stock_result.stock_result
)
except PostSaveError:
# mark the stub as saved if there's a post save error
# but re-raise the error so that the re-processing queue picks it up
unfinished_submission_stub.submission_saved()
raise
else:
unfinished_submission_stub.submission_saved()
self.do_post_save_actions(case_db, xforms, case_stock_result)
except PostSaveError:
return "Error performing post save operations"
@staticmethod
@tracer.wrap(name='submission.post_save_actions')
def do_post_save_actions(case_db, xforms, case_stock_result):
instance = xforms[0]
case_db.clear_changed()
try:
case_stock_result.case_result.commit_dirtiness_flags()
case_stock_result.stock_result.finalize()
SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models)
close_extension_cases(
case_db,
case_stock_result.case_models,
"SubmissionPost-%s-close_extensions" % instance.form_id
)
except PostSaveError:
raise
except Exception:
notify_exception(get_request(), "Error performing post save actions during form processing", {
'domain': instance.domain,
'form_id': instance.form_id,
})
raise PostSaveError
@staticmethod
@tracer.wrap(name='submission.process_cases_and_stock')
def process_xforms_for_cases(xforms, case_db):
from casexml.apps.case.xform import process_cases_with_casedb
from corehq.apps.commtrack.processing import process_stock
instance = xforms[0]
case_result = process_cases_with_casedb(xforms, case_db)
stock_result = process_stock(xforms, case_db)
modified_on_date = instance.received_on
if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on:
modified_on_date = instance.edited_on
cases = case_db.get_cases_for_saving(modified_on_date)
stock_result.populate_models()
return CaseStockProcessingResult(
case_result=case_result,
case_models=cases,
stock_result=stock_result,
)
def get_response(self):
return self.run().response
@staticmethod
def _fire_post_save_signals(instance, cases):
from casexml.apps.case.signals import case_post_save
error_message = "Error occurred during form submission post save (%s)"
error_details = {'domain': instance.domain, 'form_id': instance.form_id}
results = successful_form_received.send_robust(None, xform=instance)
has_errors = log_signal_errors(results, error_message, error_details)
for case in cases:
results = case_post_save.send_robust(case.__class__, case=case)
has_errors |= log_signal_errors(results, error_message, error_details)
if has_errors:
raise PostSaveError
def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None):
if self.is_openrosa_version3:
instance_ok = instance.is_normal or instance.is_duplicate
has_error = error_message or error_nature
if instance_ok and not has_error:
response = openrosa_response.get_openarosa_success_response(message=success_message)
else:
error_message = error_message or instance.problem
response = self.get_v3_error_response(error_message, error_nature)
else:
if instance.is_normal:
response = openrosa_response.get_openarosa_success_response()
else:
response = self.get_v2_submit_error_response(instance)
# this hack is required for ODK
response["Location"] = self.location
# this is a magic thing that we add
response['X-CommCareHQ-FormID'] = instance.form_id
return response
@staticmethod
def get_v2_submit_error_response(doc):
return OpenRosaResponse(
message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201,
).response()
@staticmethod
def get_v3_error_response(message, nature):
"""Returns a 422(Unprocessable Entity) response
- if nature == 'processing_failure' the mobile device will quarantine this form and not retry it
- any other value of `nature` will result in the form being marked as a failure and retrying
"""
return OpenRosaResponse(
message=message, nature=nature, status=422,
).response()
@staticmethod
def get_exception_response_and_log(msg, error_instance, path):
logging.error(
msg,
extra={
'submission_path': path,
'form_id': error_instance.form_id,
'error_message': error_instance.problem
}
)
# This are generally badly formed XML resulting from file corruption, encryption errors
# or other errors on the device which can not be recovered from.
# To prevent retries of these errors we submit a 422 response with `processing_failure` nature.
return OpenRosaResponse(
message="There was an error processing the form: %s" % error_instance.problem,
nature=ResponseNature.PROCESSING_FAILURE,
status=422,
).response()
@tracer.wrap(name='submission.handle_system_action')
def handle_system_action(self, form):
handle_system_action(form, self.auth_context)
self.interface.save_processed_models([form])
response = HttpResponse(status=201)
return FormProcessingResult(response, form, [], [], 'system-action')
@tracer.wrap(name='submission.process_device_log')
def process_device_log(self, device_log_form):
self._conditionally_send_device_logs_to_sumologic(device_log_form)
ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.NO_DEVICE_LOG_ENVS
if self.force_logs or not ignore_device_logs:
try:
process_device_log(self.domain, device_log_form, self.force_logs)
except Exception as e:
notify_exception(None, "Error processing device log", details={
'xml': self.instance,
'domain': self.domain
})
e.sentry_capture = False
raise
response = self._get_open_rosa_response(device_log_form)
return FormProcessingResult(response, device_log_form, [], [], 'device-log')
def setUp(self):
super(TestReprocessDuringSubmission, self).setUp()
self.factory = CaseFactory(domain=self.domain)
self.formdb = FormAccessors(self.domain)
self.casedb = CaseAccessors(self.domain)
self.ledgerdb = LedgerAccessors(self.domain)
