class TestHardDelete(TestCase): def setUp(self): self.casedb = CaseAccessors(TEST_DOMAIN_NAME) self.formdb = FormAccessors(TEST_DOMAIN_NAME) @run_with_all_backends def test_simple_delete(self): factory = CaseFactory() case = factory.create_case() [case] = factory.create_or_update_case( CaseStructure(case_id=case.case_id, attrs={'update': {'foo': 'bar'}}) ) self.assertIsNotNone(self.casedb.get_case(case.case_id)) self.assertEqual(2, len(case.xform_ids)) for form_id in case.xform_ids: self.assertIsNotNone(self.formdb.get_form(form_id)) safe_hard_delete(case) with self.assertRaises(CaseNotFound): self.casedb.get_case(case.case_id) for form_id in case.xform_ids: with self.assertRaises(XFormNotFound): self.formdb.get_form(form_id) @run_with_all_backends def test_delete_with_related(self): factory = CaseFactory() parent = factory.create_case() [child] = factory.create_or_update_case( CaseStructure(attrs={'create': True}, walk_related=False, indices=[ CaseIndex(CaseStructure(case_id=parent.case_id)) ]), ) # deleting the parent should not be allowed because the child still references it with self.assertRaises(CommCareCaseError): safe_hard_delete(parent) # deleting the child is ok safe_hard_delete(child) self.assertIsNotNone(self.casedb.get_case(parent.case_id)) with self.assertRaises(CaseNotFound): self.casedb.get_case(child.case_id) @run_with_all_backends def test_delete_sharing_form(self): factory = CaseFactory() c1, c2 = factory.create_or_update_cases([ CaseStructure(attrs={'create': True}), CaseStructure(attrs={'create': True}), ]) with self.assertRaises(CommCareCaseError): safe_hard_delete(c1) with self.assertRaises(CommCareCaseError): safe_hard_delete(c2) self.assertIsNotNone(self.casedb.get_case(c1.case_id)) self.assertIsNotNone(self.casedb.get_case(c2.case_id))
def handle(self, domain, folder_path, **options): if os.path.exists(folder_path): if not os.path.isdir(folder_path): raise CommandError('Folder path must be the path to a directory') else: os.mkdir(folder_path) form_accessors = FormAccessors(domain) form_ids = form_accessors.get_all_form_ids_in_domain() for form in form_accessors.iter_forms(form_ids): form_path = os.path.join(folder_path, form.form_id) if not os.path.exists(form_path): os.mkdir(form_path) form_meta = FormMetadata( user_id=form.user_id, received_on=form.received_on, app_id=form.app_id, build_id=form.build_id, attachments=form.attachments.keys(), auth_context=form.auth_context, ) with open(os.path.join(form_path, 'metadata.json'), 'w') as meta: json.dump(form_meta.to_json(), meta) xml = form.get_xml() with open(os.path.join(form_path, 'form.xml'), 'w') as f: f.write(xml) for name, meta in form.attachments.items(): with open(os.path.join(form_path, name), 'w') as f: f.write(form.get_attachment(name))
def test_submit_deprecated_form_with_attachments(self): def list_attachments(form): return sorted( (att.name, att.open().read()) for att in form.get_attachments() if att.name != "form.xml" ) self._submit('simple_form.xml', attachments={ "image": BytesIO(b"fake image"), "file": BytesIO(b"text file"), }) response = self._submit( 'simple_form_edited.xml', attachments={"image": BytesIO(b"other fake image")}, url=reverse("receiver_secure_post", args=[self.domain]), ) acc = FormAccessors(self.domain.name) new_form = acc.get_form(response['X-CommCareHQ-FormID']) old_form = acc.get_form(new_form.deprecated_form_id) self.assertIn(b"<bop>bang</bop>", old_form.get_xml()) self.assertIn(b"<bop>bong</bop>", new_form.get_xml()) self.assertEqual(list_attachments(old_form), [("file", b"text file"), ("image", b"fake image")]) self.assertEqual(list_attachments(new_form), [("file", b"text file"), ("image", b"other fake image")])
def _handle_duplicate(new_doc, instance): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id) existing_md5 = existing_doc.xml_md5() new_md5 = hashlib.md5(instance).hexdigest() if existing_md5 != new_md5: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return FormProcessingResult(new_doc, existing_doc) else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return FormProcessingResult(duplicate)
def test_archiving_only_form(self): """ Checks that archiving the only form associated with the case archives the case and unarchiving unarchives it. """ case_id = _post_util(create=True, p1='p1-1', p2='p2-1') case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(1, len(case.actions)) else: self.assertEqual(2, len(case.actions)) [form_id] = case.xform_ids form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id) form.archive() case = case_accessors.get_case(case_id) self.assertTrue(case.is_deleted) # should just have the 'rebuild' action self.assertEqual(1, len(case.actions)) self.assertTrue(case.actions[0].is_case_rebuild) form.unarchive() case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(3, len(case.actions)) self.assertTrue(case.actions[-1].is_case_rebuild)
def _delete_all_forms(domain_name): logger.info('Deleting forms...') form_accessor = FormAccessors(domain_name) form_ids = list(itertools.chain(*[ form_accessor.get_all_form_ids_in_domain(doc_type=doc_type) for doc_type in doc_type_to_state ])) for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500): form_accessor.soft_delete_forms(list(form_id_chunk)) logger.info('Deleting forms complete.')
def test_archive_last_form(self): initial_amounts = [(p._id, float(100)) for p in self.products] self.submit_xml_form( balance_submission(initial_amounts), timestamp=datetime.utcnow() + timedelta(-30) ) final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)] second_form_id = self.submit_xml_form(balance_submission(final_amounts)) ledger_accessors = LedgerAccessors(self.domain.name) def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual(3, len(self._get_all_ledger_transactions(Q(form_id=second_form_id)))) else: self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count()) # 6 = 3 stockonhand and 3 inferred consumption txns self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for lv in ledger_values: self.assertEqual(50, lv.stock_on_hand) self.assertEqual( round(float(lv.daily_consumption), 2), 1.67 ) # check initial setup _assert_initial_state() # archive and confirm commtrack data is deleted form = FormAccessors(self.domain.name).get_form(second_form_id) with process_pillow_changes('LedgerToElasticsearchPillow'): form.archive() if should_use_sql_backend(self.domain): self.assertEqual(0, len(self._get_all_ledger_transactions(Q(form_id=second_form_id)))) else: self.assertEqual(0, StockReport.objects.filter(form_id=second_form_id).count()) self.assertEqual(0, StockTransaction.objects.filter(report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: # balance should be reverted to 100 in the StockState self.assertEqual(100, int(state.stock_on_hand)) # consumption should be none since there will only be 1 data point self.assertIsNone(state.daily_consumption) # unarchive and confirm commtrack data is restored with process_pillow_changes('LedgerToElasticsearchPillow'): form.unarchive() _assert_initial_state()
def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain)
def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, "'domain' is required" assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db if case_db: assert case_db.domain == domain self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 self.track_load = form_load_counter("form_submission", domain)
def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission
def test_archived_form_gets_removed_from_case_xform_ids(self): initial_amounts = [(p._id, float(100)) for p in self.products] instance_id = self.submit_xml_form( balance_submission(initial_amounts), timestamp=datetime.utcnow() + timedelta(-30) ) case_accessors = CaseAccessors(self.domain.name) case = case_accessors.get_case(self.sp.case_id) self.assertIn(instance_id, case.xform_ids) form = FormAccessors(self.domain.name).get_form(instance_id) form.archive() case = case_accessors.get_case(self.sp.case_id) self.assertNotIn(instance_id, case.xform_ids)
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): return self.form_accessors.get_form(doc_id).to_json()
def test_reprocess_xform_error(self): case_id = uuid.uuid4().hex parent_case_id = uuid.uuid4().hex case = CaseBlock( create=True, case_id=case_id, user_id='user1', owner_id='user1', case_type='demo', case_name='child', index={'parent': ('parent_type', parent_case_id)} ) post_case_blocks([case.as_xml()], domain=self.domain) form_accessors = FormAccessors(self.domain) error_forms = form_accessors.get_forms_by_type('XFormError', 10) self.assertEqual(1, len(error_forms)) form = error_forms[0] reprocess_xform_error(form) error_forms = form_accessors.get_forms_by_type('XFormError', 10) self.assertEqual(1, len(error_forms)) case = CaseBlock( create=True, case_id=parent_case_id, user_id='user1', owner_id='user1', case_type='parent_type', case_name='parent', ) post_case_blocks([case.as_xml()], domain=self.domain) reprocess_xform_error(form_accessors.get_form(form.form_id)) form = form_accessors.get_form(form.form_id) # self.assertTrue(form.initial_processing_complete) Can't change this with SQL forms at the moment self.assertTrue(form.is_normal) self.assertIsNone(form.problem) case = CaseAccessors(self.domain).get_case(case_id) self.assertEqual(1, len(case.indices)) self.assertEqual(case.indices[0].referenced_id, parent_case_id) self._validate_case(case)
def test_soft_delete(self): meta = TestFormMetadata(domain=DOMAIN) get_simple_wrapped_form('f1', metadata=meta) f2 = get_simple_wrapped_form('f2', metadata=meta) f2.archive() get_simple_wrapped_form('f3', metadata=meta) accessors = FormAccessors(DOMAIN) # delete num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123') self.assertEqual(num, 2) for form_id in ['f1', 'f2']: form = accessors.get_form(form_id) self.assertTrue(form.is_deleted) self.assertEqual(form.deletion_id, '123') form = accessors.get_form('f3') self.assertFalse(form.is_deleted) for form_id in ['f1', 'f2']: form = FormAccessors(DOMAIN).get_form(form_id) if isinstance(form, DisabledDbMixin): super(DisabledDbMixin, form).delete() else: form.delete()
def test_archive_against_deleted_case(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, p1='p1', form_extras={'received_on': now}) _post_util(case_id=case_id, p2='p2', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p3='p3', form_extras={'received_on': now + timedelta(seconds=2)}) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) case_accessors.soft_delete_cases([case_id]) [f1, f2, f3] = case.xform_ids f2_doc = FormAccessors(REBUILD_TEST_DOMAIN).get_form(f2) f2_doc.archive() case = case_accessors.get_case(case_id) self.assertTrue(case.is_deleted)
def test_error_with_normal_doc_type_migration(self): submit_form_locally( """<data xmlns="example.com/foo"> <meta> <instanceID>im-a-bad-form</instanceID> </meta> <case case_id="" xmlns="http://commcarehq.org/case/transaction/v2"> <update><foo>bar</foo></update> </case> </data>""", self.domain_name, ) form = FormAccessors(self.domain_name).get_form('im-a-bad-form') form_json = form.to_json() form_json['doc_type'] = 'XFormInstance' XFormInstance.wrap(form_json).save() self._do_migration_and_assert_flags(self.domain_name) self.assertEqual(1, len(self._get_form_ids('XFormError'))) self._compare_diffs([])
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): try: return add_couch_properties_to_sql_form_json(self.form_accessors.get_form(doc_id).to_json()) except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.get_all_form_ids_in_domain()) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield add_couch_properties_to_sql_form_json(wrapped_form.to_json())
def test_update_responses(self): formxml = FormSubmissionBuilder( form_id='123', form_properties={'breakfast': 'toast', 'lunch': 'sandwich'} ).as_xml_string() pic = UploadedFile(BytesIO(b"fake"), 'pic.jpg', content_type='image/jpeg') xform = submit_form_locally(formxml, DOMAIN, attachments={"image": pic}).xform updates = {'breakfast': 'fruit'} errors = FormProcessorInterface(DOMAIN).update_responses(xform, updates, 'user1') form = FormAccessors(DOMAIN).get_form(xform.form_id) self.assertEqual(0, len(errors)) self.assertEqual('fruit', form.form_data['breakfast']) self.assertEqual('sandwich', form.form_data['lunch']) self.assertIn("image", form.attachments) self.assertEqual(form.get_attachment("image"), b"fake") self.assertXmlEqual( form.get_attachment("form.xml").decode('utf-8'), formxml.replace("toast", "fruit"), )
def _reopen_or_create_supply_point(location): from .helpers import update_supply_point_from_location from .dbaccessors import get_supply_point_by_location_id supply_point = get_supply_point_by_location_id(location.domain, location.location_id) if supply_point: if supply_point and supply_point.closed: form_ids = CaseAccessors(supply_point.domain).get_case_xform_ids(supply_point.case_id) form_accessor = FormAccessors(supply_point.domain) for form_id in form_ids: form = form_accessor.get_form(form_id) closes_case = any(map( lambda case_update: case_update.closes_case(), get_case_updates(form), )) if closes_case: form.archive(user_id=const.COMMTRACK_USERNAME) update_supply_point_from_location(supply_point, location) return supply_point else: return SupplyInterface.create_from_location(location.domain, location)
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): try: return self.form_accessors.get_form(doc_id).to_json() except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e)
def perform_resave_on_xforms(domain, start_date, end_date, no_input): _, _, xform_ids_missing_in_es, _ = compare_xforms(domain, 'XFormInstance', start_date, end_date) print("%s Ids found for xforms missing in ES." % len(xform_ids_missing_in_es)) if len(xform_ids_missing_in_es) < 1000: print(xform_ids_missing_in_es) if no_input is not True: ok = input("Type 'ok' to continue: ") if ok != "ok": print("No changes made") return form_accessor = FormAccessors(domain) for xform_ids in chunked(with_progress_bar(xform_ids_missing_in_es), 100): xforms = form_accessor.get_forms(list(xform_ids)) found_xform_ids = set() for xform in xforms: resave_form(domain, xform) found_xform_ids.add(xform.form_id) for xform_id in set(xform_ids) - found_xform_ids: print("form not found %s" % xform_id)
def handle(self, username, domain, **options): this_form_accessor = FormAccessors(domain=domain) user = CouchUser.get_by_username(username) if not user: logger.info("User {} not found.".format(username)) sys.exit(1) user_id = user._id form_ids = this_form_accessor.get_form_ids_for_user(user_id) input_response = six.moves.input( "Update {} form(s) for user {} in domain {}? (y/n): ".format(len(form_ids), username, domain)) if input_response == "y": for form_data in this_form_accessor.iter_forms(form_ids): form_attachment_xml_new = self.update_form_data(form_data, NEW_USERNAME) this_form_accessor.modify_attachment_xml_and_metadata(form_data, form_attachment_xml_new, NEW_USERNAME) logging.info("Updated {} form(s) for user {} in domain {}".format(len(form_ids), username, domain)) elif input_response == "n": logging.info("No forms updated, exiting.") else: logging.info("Command not recognized. Exiting.")
def test_form_with_missing_xmlns(self): form_id = uuid.uuid4().hex form_template = """<?xml version='1.0' ?> <data uiVersion="1" version="1" name=""{xmlns}> <name>fgg</name> <n1:meta xmlns:n1="http://openrosa.org/jr/xforms"> <n1:deviceID>354957031935664</n1:deviceID> <n1:timeStart>2016-03-01T12:04:16Z</n1:timeStart> <n1:timeEnd>2016-03-01T12:04:16Z</n1:timeEnd> <n1:username>bcdemo</n1:username> <n1:userID>user-abc</n1:userID> <n1:instanceID>{form_id}</n1:instanceID> </n1:meta> </data>""" xml = form_template.format( form_id=form_id, xmlns=' xmlns="http://openrosa.org/formdesigner/456"' ) submit_form_locally(xml, self.domain_name) # hack the form to remove XMLNS since it's now validated during form submission form = FormAccessors(self.domain_name).get_form(form_id) form.xmlns = None del form.form_data['@xmlns'] xml_no_xmlns = form_template.format(form_id=form_id, xmlns="") form.delete_attachment('form.xml') form.put_attachment(xml_no_xmlns, 'form.xml') self._do_migration_and_assert_flags(self.domain_name) self.assertEqual(1, len(self._get_form_ids())) self._compare_diffs([])
def handle(self, domains, file_name, **options): blob_db = get_blob_db() with open(file_name, 'w', encoding='utf-8') as csv_file: field_names = ['domain', 'archived', 'form_id', 'received_on'] csv_writer = csv.DictWriter(csv_file, field_names) csv_writer.writeheader() for domain in domains: self.stdout.write("Handling domain %s" % domain) form_db = FormAccessors(domain) form_ids = form_db.get_all_form_ids_in_domain() form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived')) for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)): if isinstance(form, CouchForm): meta = form.blobs.get(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) elif isinstance(form, XFormInstanceSQL): meta = form.get_attachment_meta(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) else: raise Exception("not sure how we got here")
def test_archie_modified_on(self): case_id = uuid.uuid4().hex now = datetime.utcnow().replace(microsecond=0) earlier = now - timedelta(hours=1) way_earlier = now - timedelta(days=1) # make sure we timestamp everything so they have the right order create_block = CaseBlock(case_id, create=True, date_modified=way_earlier) post_case_blocks( [create_block.as_xml()], form_extras={'received_on': way_earlier} ) update_block = CaseBlock(case_id, update={'foo': 'bar'}, date_modified=earlier) post_case_blocks( [update_block.as_xml()], form_extras={'received_on': earlier} ) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) self.assertEqual(earlier, case.modified_on) second_form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(case.xform_ids[-1]) second_form.archive() case = case_accessors.get_case(case_id) self.assertEqual(way_earlier, case.modified_on)
def test_archive_only_form(self): # check no data in stock states ledger_accessors = LedgerAccessors(self.domain.name) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(0, len(ledger_values)) initial_amounts = [(p._id, float(100)) for p in self.products] form_id = self.submit_xml_form(balance_submission(initial_amounts)) # check that we made stuff def _assert_initial_state(): if should_use_sql_backend(self.domain.name): self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: self.assertEqual(100, int(state.stock_on_hand)) _assert_initial_state() # archive and confirm commtrack data is cleared form = FormAccessors(self.domain.name).get_form(form_id) form.archive() self.assertEqual(0, len(ledger_accessors.get_ledger_values_for_case(self.sp.case_id))) if should_use_sql_backend(self.domain.name): self.assertEqual(0, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(0, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(0, StockTransaction.objects.filter(report__form_id=form_id).count()) # unarchive and confirm commtrack data is restored form.unarchive() _assert_initial_state()
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain, xmlns=None): self.domain = domain self.form_accessors = FormAccessors(domain=domain) self.xmlns = xmlns def get_document(self, doc_id): try: form = self.form_accessors.get_form(doc_id) if isinstance(form, XFormInstanceSQL): return form.to_json(include_attachments=True) else: return form.to_json() except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns)) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield wrapped_form.to_json()
def setUp(self): super(KafkaPublishingTest, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.form_accessors = FormAccessors(domain=self.domain) self.processor = TestProcessor() self.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'), processor=self.processor ) self.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'), processor=self.processor ) self.ledger_pillow = ConstructedPillow( name='test-kafka-ledger-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'), processor=self.processor )
def reprocess_archive_stubs(): # Check for archive stubs from corehq.form_processor.interfaces.dbaccessors import FormAccessors from couchforms.models import UnfinishedArchiveStub stubs = UnfinishedArchiveStub.objects.filter() datadog_gauge('commcare.unfinished_archive_stubs', len(stubs)) start = time.time() cutoff = start + timedelta(minutes=4).total_seconds() for stub in stubs: # Exit this task after 4 minutes so that the same stub isn't ever processed in multiple queues. if time.time() - start > cutoff: return xform = FormAccessors(stub.domain).get_form(form_id=stub.xform_id) # If the history wasn't updated the first time around, run the whole thing again. if not stub.history_updated: if stub.archive: xform.archive(user_id=stub.user_id) else: xform.unarchive(user_id=stub.user_id) # If the history was updated the first time around, just send the update to kafka else: xform.publish_archive_action_to_kafka(user_id=stub.user_id, archive=stub.archive)
def setUp(self): super(BaseCaseMultimediaTest, self).setUp() self.formdb = FormAccessors() FormProcessorTestUtils.delete_all_cases() FormProcessorTestUtils.delete_all_xforms()
def test_form_archiving(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, p1='p1-1', p2='p2-1', form_extras={'received_on': now}) _post_util(case_id=case_id, p2='p2-2', p3='p3-2', p4='p4-2', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p4='p4-3', p5='p5-3', close=True, form_extras={'received_on': now + timedelta(seconds=2)}) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) closed_by = case.closed_by closed_on = case.closed_on self.assertNotEqual('', closed_by) self.assertNotEqual(None, closed_on) def _check_initial_state(case): self.assertTrue(case.closed) self.assertEqual(closed_by, case.closed_by) self.assertEqual(closed_on, case.closed_on) self.assertEqual(case.get_case_property('p1'), 'p1-1') # original self.assertEqual(case.get_case_property('p2'), 'p2-2') # updated in second post self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post self.assertEqual(case.get_case_property('p4'), 'p4-3') # updated in third post self.assertEqual(case.get_case_property('p5'), 'p5-3') # new in third post if should_use_sql_backend(REBUILD_TEST_DOMAIN): # SQL stores one transaction per form self.assertEqual(3, len( primary_actions(case))) # create + update + close else: self.assertEqual(5, len( primary_actions(case))) # create + 3 updates + close _check_initial_state(case) # verify xform/action states [f1, f2, f3] = case.xform_ids if should_use_sql_backend(REBUILD_TEST_DOMAIN): [create, update, close] = case.actions self.assertEqual(f1, create.form_id) self.assertEqual(f2, update.form_id) self.assertEqual(f3, close.form_id) else: [create, u1, u2, u3, close] = case.actions self.assertEqual(f1, create.form_id) self.assertEqual(f1, u1.form_id) self.assertEqual(f2, u2.form_id) self.assertEqual(f3, u3.form_id) # todo: should this be the behavior for archiving the create form? form_acessors = FormAccessors(REBUILD_TEST_DOMAIN) f1_doc = form_acessors.get_form(f1) with capture_kafka_changes_context(topics.CASE_SQL) as change_context: f1_doc.archive() if should_use_sql_backend(case.domain): self.assertEqual([case.case_id], [change.id for change in change_context.changes])
def __call__(self, item, context=None): xforms_ids = CaseAccessors(item['domain']).get_case_xform_ids( item['_id']) forms = FormAccessors(item['domain']).get_forms(xforms_ids) f_forms = [f for f in forms if f.xmlns == self.xmlns] s_forms = sorted(f_forms, key=lambda x: x.received_on) if len(s_forms) > 0: latest_form = s_forms[-1] else: latest_form = None path_to_action_plan = 'form/action_plan/%s/action_plan' % self.section if latest_form: action_plans = latest_form.get_data(path_to_action_plan) if action_plans: action_plan_for_question = None for action_plan in action_plans: if action_plan.get('incorrect_questions', '') == self.question_id: action_plan_for_question = action_plan break if action_plan_for_question: incorrect_question = action_plan_for_question.get( 'incorrect_questions', '') responsible = ', '.join([ item.get(x.strip(), '---') for x in action_plan_for_question.get( 'action_plan_input', {}).get( 'responsible', '').split(',') ]) support = ', '.join([ item.get(x.strip(), '---') for x in action_plan_for_question.get( 'action_plan_input', {}).get( 'support', '').split(',') ]) application = Application.get(latest_form.app_id) form = application.get_form_by_xmlns(self.xmlns) question_list = application.get_questions(self.xmlns) questions = {x['value']: x for x in question_list} return { 'form_name': form.name['en'], 'section': self.section, 'timeEnd': latest_form.metadata.timeEnd, 'gap': questions.get( 'data/code_to_text/%s' % incorrect_question, {}).get('label', '---'), 'intervention_action': action_plan_for_question.get('intervention_action', '---'), 'responsible': responsible, 'support': support, 'deadline': action_plan_for_question.get('DEADLINE', '---'), 'notes': action_plan_for_question.get('notes', '---'), }
] if cases_to_delete: with open(options['filename'], 'w') as csvfile: writer = csv.writer(csvfile) headers = [ 'case id', 'case type', 'owner', 'opened by', 'app version', ] writer.writerow(headers) print(headers) for case in cases_to_delete: form = FormAccessors(domain=domain).get_form( case.xform_ids[0]) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row = [ case.case_id, case.type, cached_owner_id_to_display(case.owner_id) or case.owner_id, cached_owner_id_to_display(case.opened_by), app_version_info.build_version, ]
class CallCenterUtilsUserCaseTests(TestCase): @classmethod def setUpClass(cls): super(CallCenterUtilsUserCaseTests, cls).setUpClass() cls.domain = create_domain(TEST_DOMAIN) cls.domain.usercase_enabled = True cls.domain.save() def setUp(self): self.user = CommCareUser.create(TEST_DOMAIN, 'user1', '***', commit=False) # Don't commit yet def tearDown(self): self.user.delete() @classmethod def tearDownClass(cls): delete_all_cases() cls.domain.delete() super(CallCenterUtilsUserCaseTests, cls).tearDownClass() def test_sync_usercase_custom_user_data_on_create(self): """ Custom user data should be synced when the user is created """ self.user.user_data = { 'completed_training': 'yes', } self.user.save() case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE) self.assertIsNotNone(case) self.assertEqual(case.dynamic_case_properties()['completed_training'], 'yes') def test_sync_usercase_custom_user_data_on_update(self): """ Custom user data should be synced when the user is updated """ self.user.user_data = { 'completed_training': 'no', } self.user.save() self.user.user_data = { 'completed_training': 'yes', } sync_usercase(self.user) case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE) self.assertEqual(case.dynamic_case_properties()['completed_training'], 'yes') self._check_update_matches(case, {'completed_training': 'yes'}) def test_sync_usercase_overwrite_hq_props(self): """ Test that setting custom user data for owner_id and case_type don't change the case """ self.user.user_data = { 'owner_id': 'someone else', 'case_type': 'bob', } self.user.save() case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE) self.assertEqual(case.owner_id, self.user.get_id) self.assertEqual(case.type, USERCASE_TYPE) self.assertEqual(1, len(case.xform_ids)) def _check_update_matches(self, case, expected_update): last_form = FormAccessors(TEST_DOMAIN).get_form(case.xform_ids[-1]) case_update = get_case_updates(last_form)[0] self.assertDictEqual(case_update.update_block, expected_update)
if form.domain != domain: return False case_ids = get_case_ids_from_form(form) # all cases touched by the form and not already modified for case in CaseAccessors(domain).iter_cases(case_ids - modified_cases): if case.is_deleted != is_deletion: # we can't delete/undelete this form - this would change the state of `case` return False # all cases touched by this form are deleted return True if is_deletion or Domain.get_by_name(domain).use_sql_backend: all_forms = FormAccessors(domain).iter_forms(form_ids_to_modify) else: # accessor.iter_forms doesn't include deleted forms on the couch backend all_forms = list( map(FormAccessors(domain).get_form, form_ids_to_modify)) return [form.form_id for form in all_forms if _is_safe_to_modify(form)] @task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True) def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date): to_delete = _get_forms_to_modify(domain, deleted_forms,
def test_archive_last_form(self): initial_amounts = [(p._id, float(100)) for p in self.products] self.submit_xml_form(balance_submission(initial_amounts), timestamp=datetime.utcnow() + timedelta(-30)) final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)] second_form_id = self.submit_xml_form( balance_submission(final_amounts)) ledger_accessors = LedgerAccessors(self.domain.name) def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual( 3, len( self._get_all_ledger_transactions( Q(form_id=second_form_id)))) else: self.assertEqual( 1, StockReport.objects.filter(form_id=second_form_id).count()) # 6 = 3 stockonhand and 3 inferred consumption txns self.assertEqual( 6, StockTransaction.objects.filter( report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case( self.sp.case_id) self.assertEqual(3, len(ledger_values)) for lv in ledger_values: self.assertEqual(50, lv.stock_on_hand) self.assertEqual(round(float(lv.daily_consumption), 2), 1.67) # check initial setup _assert_initial_state() # archive and confirm commtrack data is deleted form = FormAccessors(self.domain.name).get_form(second_form_id) with process_pillow_changes('LedgerToElasticsearchPillow'): form.archive() if should_use_sql_backend(self.domain): self.assertEqual( 0, len( self._get_all_ledger_transactions( Q(form_id=second_form_id)))) else: self.assertEqual( 0, StockReport.objects.filter(form_id=second_form_id).count()) self.assertEqual( 0, StockTransaction.objects.filter( report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case( self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: # balance should be reverted to 100 in the StockState self.assertEqual(100, int(state.stock_on_hand)) # consumption should be none since there will only be 1 data point self.assertIsNone(state.daily_consumption) # unarchive and confirm commtrack data is restored with process_pillow_changes('LedgerToElasticsearchPillow'): form.unarchive() _assert_initial_state()
def payload_doc(self, repeat_record): return FormAccessors(repeat_record.domain).get_form(repeat_record.payload_id)
def test_get_payload(self): self.post_xml(self.xform_xml, self.domain) payload_doc = FormAccessors(self.domain).get_form(self.instance_id) payload = self.repeatergenerator.get_payload(None, payload_doc) self.assertXmlEqual(self.xform_xml, payload)
def _get_forms_to_archive(self): # ordered with latest form's id on top form_accessor = FormAccessors(self.domain) form_ids = form_accessor.get_form_ids_for_user(self.user_id) return [f for f in form_accessor.get_forms(form_ids) if f.is_normal]
def setUp(self): super(EditFormTest, self).setUp() self.interface = FormProcessorInterface(self.domain) self.casedb = CaseAccessors(self.domain) self.formdb = FormAccessors(self.domain)
class EditFormTest(TestCase, TestFileMixin): ID = '7H46J37FGH3' domain = 'test-form-edits' file_path = ('data', 'deprecation') root = os.path.dirname(__file__) def setUp(self): super(EditFormTest, self).setUp() self.interface = FormProcessorInterface(self.domain) self.casedb = CaseAccessors(self.domain) self.formdb = FormAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_xforms(self.domain) FormProcessorTestUtils.delete_all_cases(self.domain) UnfinishedSubmissionStub.objects.all().delete() super(EditFormTest, self).tearDown() def test_basic_edit(self): original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') xform = submit_form_locally(original_xml, self.domain).xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual("", xform.form_data['vitals']['height']) self.assertEqual("other", xform.form_data['assessment']['categories']) xform = submit_form_locally(edit_xml, self.domain).xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual("100", xform.form_data['vitals']['height']) self.assertEqual("Edited Baby!", xform.form_data['assessment']['categories']) self.assertEqual(1, len(xform.history)) self.assertEqual('edit', xform.history[0].operation) deprecated_xform = self.formdb.get_form(xform.deprecated_form_id) self.assertEqual(self.ID, deprecated_xform.orig_id) self.assertNotEqual(self.ID, deprecated_xform.form_id) self.assertTrue(deprecated_xform.is_deprecated) self.assertEqual("", deprecated_xform.form_data['vitals']['height']) self.assertEqual( "other", deprecated_xform.form_data['assessment']['categories']) self.assertEqual(xform.received_on, deprecated_xform.received_on) self.assertEqual(xform.deprecated_form_id, deprecated_xform.form_id) self.assertTrue(xform.edited_on > deprecated_xform.received_on) self.assertEqual(deprecated_xform.get_xml().decode('utf-8'), original_xml.decode('utf-8')) self.assertEqual(xform.get_xml().decode('utf-8'), edit_xml.decode('utf-8')) def test_edit_form_with_attachments(self): attachment_source = './corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg' attachment_file = open(attachment_source, 'rb') attachments = { 'fruity_file': UploadedFile(attachment_file, 'fruity_file', content_type='image/jpeg') } def _get_xml(date, form_id): return """<?xml version='1.0' ?> <data uiVersion="1" version="1" name="" xmlns="http://openrosa.org/formdesigner/123"> <name>fgg</name> <date>2011-06-07</date> <n1:meta xmlns:n1="http://openrosa.org/jr/xforms"> <n1:deviceID>354957031935664</n1:deviceID> <n1:timeStart>{date}</n1:timeStart> <n1:timeEnd>{date}</n1:timeEnd> <n1:username>bcdemo</n1:username> <n1:userID>user-abc</n1:userID> <n1:instanceID>{form_id}</n1:instanceID> </n1:meta> </data>""".format(date=date, attachment_source=attachment_source, form_id=form_id) form_id = uuid.uuid4().hex original_xml = _get_xml('2016-03-01T12:04:16Z', form_id) submit_form_locally( original_xml, self.domain, attachments=attachments, ) form = self.formdb.get_form(form_id) self.assertIn('fruity_file', form.attachments) self.assertIn(original_xml, form.get_xml().decode('utf-8')) # edit form edit_xml = _get_xml('2016-04-01T12:04:16Z', form_id) submit_form_locally( edit_xml, self.domain, ) form = self.formdb.get_form(form_id) self.assertIsNotNone(form.edited_on) self.assertIsNotNone(form.deprecated_form_id) self.assertIn('fruity_file', form.attachments) self.assertIn(edit_xml, form.get_xml().decode('utf-8')) def test_edit_an_error(self): form_id = uuid.uuid4().hex case_block = CaseBlock.deprecated_init( create=True, case_id='', # this should cause the submission to error case_type='person', owner_id='some-owner', ) form, _ = submit_case_blocks(case_block.as_text(), domain=self.domain, form_id=form_id) self.assertTrue(form.is_error) self.assertTrue('IllegalCaseId' in form.problem) case_block.case_id = uuid.uuid4().hex form, _ = submit_case_blocks(case_block.as_text(), domain=self.domain, form_id=form_id) self.assertFalse(form.is_error) self.assertEqual(None, getattr(form, 'problem', None)) def test_broken_save(self): """ Test that if the second form submission terminates unexpectedly and the main form isn't saved, then there are no side effects such as the original having been marked as deprecated. """ original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') result = submit_form_locally(original_xml, self.domain) xform = result.xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual(self.domain, xform.domain) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 0) with patch.object(self.interface.processor, 'save_processed_models', side_effect=HTTPError): with self.assertRaises(HTTPError): submit_form_locally(edit_xml, self.domain) xform = self.formdb.get_form(self.ID) self.assertIsNotNone(xform) # it didn't go through, so make sure there are no edits still self.assertIsNone(getattr(xform, 'deprecated_form_id', None)) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 0) def test_case_management(self): form_id = uuid.uuid4().hex case_id = uuid.uuid4().hex owner_id = uuid.uuid4().hex case_block = CaseBlock.deprecated_init(create=True, case_id=case_id, case_type='person', owner_id=owner_id, update={ 'property': 'original value' }).as_text() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) # validate some assumptions case = self.casedb.get_case(case_id) self.assertEqual(case.type, 'person') self.assertEqual(case.dynamic_case_properties()['property'], 'original value') self.assertEqual([form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual(2, len(case.actions)) for a in case.actions: self.assertEqual(form_id, a.xform_id) # submit a new form with a different case update case_block = CaseBlock.deprecated_init(create=True, case_id=case_id, case_type='newtype', owner_id=owner_id, update={ 'property': 'edited value' }).as_text() xform, _ = submit_case_blocks(case_block, domain=self.domain, form_id=form_id) case = self.casedb.get_case(case_id) self.assertEqual(case.type, 'newtype') self.assertEqual(case.dynamic_case_properties()['property'], 'edited value') self.assertEqual([form_id], case.xform_ids) self.assertEqual(case.server_modified_on, xform.edited_on) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual(2, len(case.actions)) for a in case.actions: self.assertEqual(form_id, a.xform_id) def test_second_edit_fails(self): form_id = uuid.uuid4().hex case_id = uuid.uuid4().hex case_block = CaseBlock.deprecated_init( create=True, case_id=case_id, case_type='person', ).as_text() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) # submit an edit form with a bad case update (for example a bad ID) case_block = CaseBlock.deprecated_init( create=True, case_id='', case_type='person', ).as_text() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) xform = self.formdb.get_form(form_id) self.assertTrue(xform.is_error) deprecated_xform = self.formdb.get_form(xform.deprecated_form_id) self.assertTrue(deprecated_xform.is_deprecated) def test_case_management_ordering(self): case_id = uuid.uuid4().hex owner_id = uuid.uuid4().hex # create a case case_block = CaseBlock.deprecated_init( create=True, case_id=case_id, case_type='person', owner_id=owner_id, ).as_text() create_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual([create_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue( create_form_id in [a.xform_id for a in case.actions]) for a in case.actions: self.assertEqual(create_form_id, a.xform_id) edit_date = datetime.utcnow() # set some property value case_block = CaseBlock.deprecated_init(create=False, case_id=case_id, date_modified=edit_date, update={ 'property': 'first value', }).as_text() edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'first value') self.assertEqual([create_form_id, edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue( all(form_id in [a.xform_id for a in case.actions] for form_id in [create_form_id, edit_form_id])) # submit a second (new) form updating the value case_block = CaseBlock.deprecated_init(create=False, case_id=case_id, update={ 'property': 'final value', }).as_text() second_edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'final value') self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue( all(form_id in [a.xform_id for a in case.actions] for form_id in [create_form_id, edit_form_id, second_edit_form_id])) # deprecate the middle edit case_block = CaseBlock.deprecated_init( create=False, case_id=case_id, date_modified= edit_date, # need to use the previous edit date for action sort comparisons update={ 'property': 'edited value', 'added_property': 'added value', }).as_text() submit_case_blocks(case_block, domain=self.domain, form_id=edit_form_id) # ensure that the middle edit stays in the right place and is applied # before the final one case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'final value') self.assertEqual(case.dynamic_case_properties()['added_property'], 'added value') self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual([ create_form_id, create_form_id, edit_form_id, second_edit_form_id ], [a.xform_id for a in case.actions]) def test_edit_different_xmlns(self): form_id = uuid.uuid4().hex case1_id = uuid.uuid4().hex case2_id = uuid.uuid4().hex xmlns1 = 'http://commcarehq.org/xmlns1' xmlns2 = 'http://commcarehq.org/xmlns2' case_block = CaseBlock.deprecated_init( create=True, case_id=case1_id, case_type='person', owner_id='owner1', ).as_text() xform, cases = submit_case_blocks(case_block, domain=self.domain, xmlns=xmlns1, form_id=form_id) self.assertTrue(xform.is_normal) self.assertEqual(form_id, xform.form_id) case_block = CaseBlock.deprecated_init( create=True, case_id=case2_id, case_type='goat', owner_id='owner1', ).as_text() # submit new form with same form ID but different XMLNS xform, cases = submit_case_blocks(case_block, domain=self.domain, xmlns=xmlns2, form_id=form_id) self.assertTrue(xform.is_normal) self.assertNotEqual(form_id, xform.form_id) # form should have a different ID def test_copy_operations(self): original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') xform = submit_form_locally(original_xml, self.domain).xform xform.archive(user_id='user1') xform.unarchive(user_id='user2') xform = submit_form_locally(edit_xml, self.domain).xform self.assertEqual(3, len(xform.history)) self.assertEqual('archive', xform.history[0].operation) self.assertEqual('unarchive', xform.history[1].operation) self.assertEqual('edit', xform.history[2].operation)
class TestReprocessDuringSubmission(TestCase): @classmethod def setUpClass(cls): super(TestReprocessDuringSubmission, cls).setUpClass() cls.domain = uuid.uuid4().hex def setUp(self): super(TestReprocessDuringSubmission, self).setUp() self.factory = CaseFactory(domain=self.domain) self.formdb = FormAccessors(self.domain) self.casedb = CaseAccessors(self.domain) self.ledgerdb = LedgerAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) super(TestReprocessDuringSubmission, self).tearDown() def test_error_saving(self): case_id = uuid.uuid4().hex form_id = uuid.uuid4().hex with _patch_save_to_raise_error(self): submit_case_blocks(CaseBlock(case_id=case_id, create=True, case_type='box').as_text(), self.domain, form_id=form_id) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form = self.formdb.get_form(form_id) self.assertTrue(form.is_error) with self.assertRaises(CaseNotFound): self.casedb.get_case(case_id) result = submit_form_locally( instance=form.get_xml(), domain=self.domain, ) duplicate_form = result.xform self.assertTrue(duplicate_form.is_duplicate) case = self.casedb.get_case(case_id) self.assertIsNotNone(case) form = self.formdb.get_form(form_id) self.assertTrue(form.is_normal) self.assertIsNone(getattr(form, 'problem', None)) self.assertEqual(duplicate_form.orig_id, form.form_id) def test_processing_error(self): case_id = uuid.uuid4().hex parent_case_id = uuid.uuid4().hex form_id = uuid.uuid4().hex form, _ = submit_case_blocks(CaseBlock( case_id=case_id, create=True, case_type='box', index={ 'cupboard': ('cupboard', parent_case_id) }, ).as_text(), self.domain, form_id=form_id) self.assertTrue(form.is_error) self.assertTrue('InvalidCaseIndex' in form.problem) self.assertEqual(form.form_id, form_id) with self.assertRaises(CaseNotFound): self.casedb.get_case(case_id) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(0, len(stubs)) # create parent case submit_case_blocks( CaseBlock(case_id=parent_case_id, create=True, case_type='cupboard').as_text(), self.domain, ) # re-submit form result = submit_form_locally( instance=form.get_xml(), domain=self.domain, ) duplicate_form = result.xform self.assertTrue(duplicate_form.is_duplicate) case = self.casedb.get_case(case_id) self.assertIsNotNone(case) form = self.formdb.get_form(form_id) self.assertTrue(form.is_normal) self.assertIsNone(getattr(form, 'problem', None)) self.assertEqual(duplicate_form.orig_id, form.form_id)
def get_last_form_submission(self): result = FormAccessors(self.domain).get_forms_by_type( 'XFormInstance', 1, recent_first=True) return result[0] if len(result) > 0 else None
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.auth_context = self.auth_context.to_json() xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form instance = _transform_instance_to_error(self.interface, error, instance) xforms[0] = instance # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message) def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def run(self): failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], []) result = process_xform_xml(self.domain, self.instance, self.attachments) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form.user_id) if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = self.get_exception_response_and_log( submitted_form, self.path) return FormProcessingResult(response, None, [], []) cases = [] ledgers = [] with result.get_locked_forms() as xforms: from casexml.apps.case.xform import get_and_check_xform_domain domain = get_and_check_xform_domain(xforms[0]) if self.case_db: assert self.case_db.domain == domain case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) with case_db_cache as case_db: instance = xforms[0] if instance.xmlns == DEVICE_LOG_XMLNS: try: process_device_log(self.domain, instance) except Exception: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) raise elif instance.is_duplicate: self.interface.save_processed_models([instance]) elif not instance.is_error: try: case_stock_result = self.process_xforms_for_cases( xforms, case_db) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex) as e: self._handle_known_error(e, instance, xforms) except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True self.save_processed_models(xforms, case_stock_result) case_stock_result.case_result.close_extensions(case_db) cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save errors = self.process_signals(instance) response = self._get_open_rosa_response(instance, errors) return FormProcessingResult(response, instance, cases, ledgers) @property def _cache(self): return get_redis_default_cache() @property def _restore_cache_key(self): from casexml.apps.phone.restore import restore_cache_key return restore_cache_key def _invalidate_caches(self, user_id): """invalidate cached initial restores""" initial_restore_cache_key = self._restore_cache_key( RESTORE_CACHE_KEY_PREFIX, user_id, version=V2) self._cache.delete(initial_restore_cache_key) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_caches(user_id) def _invalidate_async_caches(self, user_id): cache_key = self._restore_cache_key(ASYNC_RESTORE_CACHE_KEY_PREFIX, user_id) task_id = self._cache.get(cache_key) if task_id is not None: revoke_celery_task(task_id) self._cache.delete(cache_key) def save_processed_models(self, xforms, case_stock_result): from casexml.apps.case.signals import case_post_save instance = xforms[0] with unfinished_submission(instance) as unfinished_submission_stub: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result) unfinished_submission_stub.saved = True unfinished_submission_stub.save() case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() for case in case_stock_result.case_models: case_post_save.send(case.__class__, case=case) def process_xforms_for_cases(self, xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(instance.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response def process_signals(self, instance): feedback = successful_form_received.send_robust(None, xform=instance) errors = [] for func, resp in feedback: if resp and isinstance(resp, Exception): error_message = unicode(resp) logging.error((u"Receiver app: problem sending " u"post-save signal %s for xform %s: %s: %s") % (func, instance.form_id, type(resp).__name__, error_message)) errors.append(error_message) if errors: self.interface.xformerror_from_xform_instance( instance, ", ".join(errors)) self.formdb.update_form_problem_and_state(instance) return errors def _get_open_rosa_response(self, instance, errors): if instance.is_normal: response = self.get_success_response(instance, errors) else: response = self.get_failure_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_success_response(doc, errors): if errors: response = OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() else: response = OpenRosaResponse( # would have done ✓ but our test Nokias' fonts don't have that character message=u' √ ', nature=ResponseNature.SUBMIT_SUCCESS, status=201, ).response() return response @staticmethod def submission_ignored_response(): return OpenRosaResponse( # would have done ✓ but our test Nokias' fonts don't have that character message=u'√ (this submission was ignored)', nature=ResponseNature.SUBMIT_SUCCESS, status=201, ).response() @staticmethod def get_failure_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_exception_response_and_log(error_instance, path): logging.exception( u"Problem receiving submission to %s. Doc id: %s, Error %s" % (path, error_instance.form_id, error_instance.problem)) return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.SUBMIT_ERROR, status=500, ).response() @staticmethod def get_blacklisted_response(): return OpenRosaResponse( message=( "This submission was blocked because of an unusual volume " "of submissions from this project space. Please contact " "support to resolve."), nature=ResponseNature.SUBMIT_ERROR, status=509, ).response()
session_name = u'{app} > {form}'.format( app=app.name, form=form_name, ) if case_id: case = CaseAccessors(domain).get_case(case_id) session_name = u'{0} - {1}'.format(session_name, case.name) root_context = { 'form_url': form_url, 'formplayer_url': settings.FORMPLAYER_URL, } if instance_id: try: root_context['instance_xml'] = FormAccessors(domain).get_form( instance_id).get_xml() except XFormNotFound: raise Http404() session_extras = {'session_name': session_name, 'app_id': app._id} session_extras.update( get_cloudcare_session_data(domain, form, request.couch_user)) delegation = request.GET.get('task-list') == 'true' session_helper = CaseSessionDataHelper(domain, request.couch_user, case_id, app, form, delegation=delegation) return json_response(
def handle_unexpected_error(interface, instance, exception): instance = _transform_instance_to_error(interface, exception, instance) notify_submission_error(instance, exception, instance.problem) FormAccessors(interface.domain).save_new_form(instance)
def setUp(self): super(FundamentalBaseTests, self).setUp() self.interface = FormProcessorInterface() self.casedb = CaseAccessors(DOMAIN) self.formdb = FormAccessors(DOMAIN)
class Command(BaseCommand): help = "Delete all cases that are in a specific case's network/footprint" def add_arguments(self, parser): parser.add_argument('domain', type=unicode) parser.add_argument('case_id', type=unicode) parser.add_argument('--filename', dest='filename', default='case-delete-info.csv') def handle(self, *args, **options): domain = options['domain'] case_id = options['case_id'] case_accessor = CaseAccessors(domain=domain) case = case_accessor.get_case(case_id) if not case.is_deleted and raw_input('\n'.join([ 'Case {} is not already deleted. Are you sure you want to delete it? (y/N)' .format(case_id) ])).lower() != 'y': sys.exit(0) dependent_case_ids = get_entire_case_network(domain, [case_id]) cases_to_delete = filter(lambda case: not case.is_deleted, case_accessor.get_cases(dependent_case_ids)) if cases_to_delete: with open(options['filename'], 'w') as csvfile: writer = csv.writer(csvfile) headers = [ 'case id', 'case type', 'owner', 'opened by', 'app version', ] writer.writerow(headers) print headers for case in cases_to_delete: form = FormAccessors(domain=domain).get_form( case.xform_ids[0]) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row = [ case.case_id, case.type, cached_owner_id_to_display(case.owner_id) or case.owner_id, cached_owner_id_to_display(case.opened_by), app_version_info.build_version, ] writer.writerow(row) print row if cases_to_delete and raw_input('\n'.join([ 'Delete these {} cases? (y/N)'.format(len(cases_to_delete)), ])).lower() == 'y': case_accessor.soft_delete_cases( [c.case_id for c in cases_to_delete]) print 'deleted {} cases'.format(len(cases_to_delete)) if cases_to_delete: print 'details here: {}'.format(options['filename']) else: print "didn't find any cases to delete"
def get_forms(self): from corehq.apps.api.util import form_to_es_form forms = FormAccessors(self.domain).get_forms(self.xform_ids) return list(filter(None, [form_to_es_form(form) for form in forms]))
class TestFormArchiving(TestCase, TestFileMixin): file_path = ('data', 'sample_xforms') root = os.path.dirname(__file__) def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain') def tearDown(self): FormProcessorTestUtils.delete_all_xforms() FormProcessorTestUtils.delete_all_cases() super(TestFormArchiving, self).tearDown() def testArchive(self): case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.archive(user_id='mr. librarian') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_archived) case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) self.assertEqual(case.xform_ids, []) [archival] = xform.history self.assertTrue(lower_bound <= archival.date <= upper_bound) self.assertEqual('archive', archival.operation) self.assertEqual('mr. librarian', archival.user) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.unarchive(user_id='mr. researcher') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_normal) case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(case.xform_ids, [xform.form_id]) [archival, restoration] = xform.history self.assertTrue(lower_bound <= restoration.date <= upper_bound) self.assertEqual('unarchive', restoration.operation) self.assertEqual('mr. researcher', restoration.user) def testSignal(self): global archive_counter, restore_counter archive_counter = 0 restore_counter = 0 def count_archive(**kwargs): global archive_counter archive_counter += 1 def count_unarchive(**kwargs): global restore_counter restore_counter += 1 xform_archived.connect(count_archive) xform_unarchived.connect(count_unarchive) xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) self.assertEqual(0, archive_counter) self.assertEqual(0, restore_counter) result.xform.archive() self.assertEqual(1, archive_counter) self.assertEqual(0, restore_counter) xform = self.formdb.get_form(result.xform.form_id) xform.unarchive() self.assertEqual(1, archive_counter) self.assertEqual(1, restore_counter)
def inaccessible_forms_accessed(self, xform_ids, domain, couch_user): xforms = FormAccessors(domain).get_forms(xform_ids) xforms_user_ids = set([xform.user_id for xform in xforms]) accessible_user_ids = set(user_ids_at_accessible_locations(domain, couch_user)) return xforms_user_ids - accessible_user_ids
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union(get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get(xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))
class ReprocessSubmissionStubTests(TestCase): @classmethod def setUpClass(cls): super(ReprocessSubmissionStubTests, cls).setUpClass() cls.domain = uuid.uuid4().hex cls.product = SQLProduct.objects.create(domain=cls.domain, product_id='product1', name='product1') @classmethod def tearDownClass(cls): cls.product.delete() super(ReprocessSubmissionStubTests, cls).tearDownClass() def setUp(self): super(ReprocessSubmissionStubTests, self).setUp() self.factory = CaseFactory(domain=self.domain) self.formdb = FormAccessors(self.domain) self.casedb = CaseAccessors(self.domain) self.ledgerdb = LedgerAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) super(ReprocessSubmissionStubTests, self).tearDown() def test_reprocess_unfinished_submission_case_create(self): case_id = uuid.uuid4().hex with _patch_save_to_raise_error(self): self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={ 'case_type': 'parent', 'create': True }) ]) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) # form that was saved before case error raised normal_form_ids = self.formdb.get_all_form_ids_in_domain( 'XFormInstance') self.assertEqual(0, len(normal_form_ids)) # shows error form (duplicate of form that was saved before case error) # this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed # the form didn't get saved # we don't really care about this form in this test error_forms = self.formdb.get_forms_by_type('XFormError', 10) self.assertEqual(1, len(error_forms)) self.assertIsNone(error_forms[0].orig_id) self.assertEqual(error_forms[0].form_id, stubs[0].xform_id) self.assertEqual(0, len(self.casedb.get_case_ids_in_domain())) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) case_ids = self.casedb.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) self.assertEqual(case_id, case_ids[0]) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk) def test_reprocess_unfinished_submission_case_update(self): case_id = uuid.uuid4().hex form_ids = [] form_ids.append( submit_case_blocks( CaseBlock(case_id=case_id, create=True, case_type='box').as_text(), self.domain)[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( CaseBlock(case_id=case_id, update={ 'prop': 'a' }).as_text(), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit second form with case update form_ids.append( submit_case_blocks( CaseBlock(case_id=case_id, update={ 'prop': 'b' }).as_text(), self.domain)[0].form_id) case = self.casedb.get_case(case_id) self.assertEqual(2, len(case.xform_ids)) self.assertEqual('b', case.get_case_property('prop')) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(0, len(result.ledgers)) case = self.casedb.get_case(case_id) self.assertEqual('b', case.get_case_property( 'prop')) # should be property value from most recent form self.assertEqual(3, len(case.xform_ids)) self.assertEqual(form_ids, case.xform_ids) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk) def test_reprocess_unfinished_submission_ledger_create(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={ 'case_type': 'parent', 'create': True }) ]) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 100), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(0, len(ledgers)) case = self.casedb.get_case(case_id) self.assertEqual(1, len(case.xform_ids)) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) self.assertEqual(0, len(ledger_transactions)) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) self.assertEqual(1, len(ledger_transactions)) # case still only has 2 transactions case = self.casedb.get_case(case_id) self.assertEqual(2, len(case.xform_ids)) if should_use_sql_backend(self.domain): self.assertTrue(case.actions[1].is_ledger_transaction) def test_reprocess_unfinished_submission_ledger_rebuild(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex form_ids = [] form_ids.append( submit_case_blocks([ CaseBlock(case_id=case_id, create=True, case_type='shop').as_text(), get_single_balance_block(case_id, 'product1', 100), ], self.domain)[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 50), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit another form afterwards form_ids.append( submit_case_blocks( get_single_balance_block(case_id, 'product1', 25), self.domain)[0].form_id) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) if should_use_sql_backend(self.domain): self.assertEqual(2, len(ledger_transactions)) else: # includes extra consumption transaction self.assertEqual(3, len(ledger_transactions)) # should rebuild ledger transactions result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) # still only 1 self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) if should_use_sql_backend(self.domain): self.assertEqual(3, len(ledger_transactions)) # make sure transactions are in correct order self.assertEqual(form_ids, [trans.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].updated_balance) self.assertEqual(100, ledger_transactions[0].delta) self.assertEqual(50, ledger_transactions[1].updated_balance) self.assertEqual(-50, ledger_transactions[1].delta) self.assertEqual(25, ledger_transactions[2].updated_balance) self.assertEqual(-25, ledger_transactions[2].delta) else: self.assertEqual(3, len(ledger_transactions)) self.assertEqual( form_ids, [trans.report.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].stock_on_hand) self.assertEqual(50, ledger_transactions[1].stock_on_hand) self.assertEqual(25, ledger_transactions[2].stock_on_hand) def test_fire_signals(self): from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler case_id = uuid.uuid4().hex form_id = uuid.uuid4().hex with failing_signal_handler('signal death'): submit_case_blocks(CaseBlock(case_id=case_id, create=True, case_type='box').as_text(), self.domain, form_id=form_id) form = self.formdb.get_form(form_id) with catch_signal( successful_form_received) as form_handler, catch_signal( case_post_save) as case_handler: submit_form_locally( instance=form.get_xml(), domain=self.domain, ) case = self.casedb.get_case(case_id) if should_use_sql_backend(self.domain): self.assertEqual(form, form_handler.call_args[1]['xform']) self.assertEqual(case, case_handler.call_args[1]['case']) else: signal_form = form_handler.call_args[1]['xform'] self.assertEqual(form.form_id, signal_form.form_id) self.assertEqual(form.get_rev, signal_form.get_rev) signal_case = case_handler.call_args[1]['case'] self.assertEqual(case.case_id, signal_case.case_id) self.assertEqual(case.get_rev, signal_case.get_rev)
class BaseCaseMultimediaTest(TestCase, TestFileMixin): file_path = ('data', 'multimedia') root = os.path.dirname(__file__) def setUp(self): super(BaseCaseMultimediaTest, self).setUp() self.formdb = FormAccessors() FormProcessorTestUtils.delete_all_cases() FormProcessorTestUtils.delete_all_xforms() def _formatXForm(self, doc_id, raw_xml, attachment_block, date=None): if date is None: date = datetime.utcnow() final_xml = Template(raw_xml.decode("utf8")).render( Context({ "attachments": attachment_block, "time_start": json_format_datetime(date - timedelta(minutes=4)), "time_end": json_format_datetime(date), "date_modified": json_format_datetime(date), "doc_id": doc_id })).encode("utf8") return final_xml def _prepAttachments(self, new_attachments, removes=[]): """ Returns: attachment_block - An XML representation of the attachment dict_attachments - A key-value dict where the key is the name and the value is a Stream of the attachment """ attachment_block = ''.join( [self._singleAttachBlock(x) for x in new_attachments] + [self._singleAttachRemoveBlock(x) for x in removes]) dict_attachments = dict( (MEDIA_FILES[attach_name], self._attachmentFileStream(attach_name)) for attach_name in new_attachments) return attachment_block, dict_attachments def _singleAttachBlock(self, key): return '<n0:%s src="%s" from="local"/>' % (key, MEDIA_FILES[key]) def _singleAttachRemoveBlock(self, key): return '<n0:%s />' % key def _attachmentFileStream(self, key): attachment_path = MEDIA_FILES[key] attachment = open(attachment_path, 'rb') return NoClose(UploadedFile(attachment, key)) def _calc_file_hash(self, key): with open(MEDIA_FILES[key], 'rb') as attach: return hashlib.md5(attach.read()).hexdigest() def _do_submit(self, xml_data, dict_attachments, sync_token=None, date=None): """ RequestFactory submitter - simulates direct submission to server directly (no need to call process case after fact) """ with flag_enabled('MM_CASE_PROPERTIES'): result = submit_form_locally(xml_data, TEST_DOMAIN_NAME, attachments=dict_attachments, last_sync_token=sync_token, received_on=date) xform = result.xform attachments = xform.attachments self.assertEqual(set(dict_attachments.keys()), set(attachments.keys())) self.assertEqual(result.case.case_id, TEST_CASE_ID) return result.response, self.formdb.get_form( xform.form_id), result.cases def _submit_and_verify(self, doc_id, xml_data, dict_attachments, sync_token=None, date=None): response, form, [case] = self._do_submit(xml_data, dict_attachments, sync_token, date=date) attachments = form.attachments self.assertEqual(len(dict_attachments), len(attachments)) for k, vstream in dict_attachments.items(): fileback = form.get_attachment(k) # rewind the pointer before comparing orig_attachment = vstream orig_attachment.seek(0) self.assertEqual( hashlib.md5(fileback).hexdigest(), hashlib.md5(orig_attachment.read()).hexdigest()) case = CaseAccessors(TEST_DOMAIN_NAME).get_case( case.case_id) # re-fetch case return form, case def _doCreateCaseWithMultimedia(self, attachments=['fruity_file']): xml_data = self.get_xml('multimedia_create') attachment_block, dict_attachments = self._prepAttachments(attachments) final_xml = self._formatXForm(CREATE_XFORM_ID, xml_data, attachment_block) return self._submit_and_verify(CREATE_XFORM_ID, final_xml, dict_attachments) def _doSubmitUpdateWithMultimedia(self, new_attachments=None, removes=None, sync_token=None, date=None): new_attachments = new_attachments if new_attachments is not None \ else ['commcare_logo_file', 'dimagi_logo_file'] removes = removes if removes is not None else ['fruity_file'] attachment_block, dict_attachments = self._prepAttachments( new_attachments, removes=removes) raw_xform = self.get_xml('multimedia_update') doc_id = uuid.uuid4().hex final_xform = self._formatXForm(doc_id, raw_xform, attachment_block, date) return self._submit_and_verify(doc_id, final_xform, dict_attachments, sync_token, date=date)
def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain')
def archive_or_restore_forms(domain, user_id, username, form_ids, archive_or_restore, task=None, from_excel=False): response = { 'errors': [], 'success': [], } missing_forms = set(form_ids) success_count = 0 if task: DownloadBase.set_progress(task, 0, len(form_ids)) for xform in FormAccessors(domain).iter_forms(form_ids): missing_forms.discard(xform.form_id) if xform.domain != domain: response['errors'].append( _("XForm {form_id} does not belong to domain {domain}").format( form_id=xform.form_id, domain=domain)) continue xform_string = _( "XForm {form_id} for domain {domain} by user '{username}'").format( form_id=xform.form_id, domain=xform.domain, username=username) try: if archive_or_restore.is_archive_mode(): xform.archive(user_id=user_id) message = _("Successfully archived {form}").format( form=xform_string) else: xform.unarchive(user_id=user_id) message = _("Successfully unarchived {form}").format( form=xform_string) response['success'].append(message) success_count = success_count + 1 except Exception as e: response['errors'].append( _("Could not archive {form}: {error}").format( form=xform_string, error=e)) if task: DownloadBase.set_progress(task, success_count, len(form_ids)) for missing_form_id in missing_forms: response['errors'].append( _("Could not find XForm {form_id}").format( form_id=missing_form_id)) if from_excel: return response response["success_count_msg"] = _("{success_msg} {count} form(s)".format( success_msg=archive_or_restore.success_text, count=success_count)) return {"messages": response}
def _get_form_attachment_info(domain, form_ids, export): properties = _get_export_properties(export) return [ _extract_form_attachment_info(form, properties) for form in FormAccessors(domain).iter_forms(form_ids) ]
def handle(self, **options): domain = options['domain'] debug = options['debug'] cleanup = options['cleanup'] domain_query = CaseES().domain(domain) valid_case_ids = set(domain_query.get_ids()) referenced_case_ids = { index['referenced_id'] for hit in domain_query.source('indices.referenced_id').run().hits for index in hit['indices'] } invalid_referenced_ids = referenced_case_ids - valid_case_ids if len(invalid_referenced_ids) > ES_MAX_CLAUSE_COUNT: print("there's a lot of invalid ids here. ES queries may not handle this well") cases_with_invalid_references = ( domain_query .term('indices.referenced_id', invalid_referenced_ids) .source(['_id', 'type', 'indices', 'owner_id', 'opened_by', 'xform_ids']) .run().hits ) with open(options['filename'], 'w', encoding='utf-8') as csvfile: writer = csv.writer(csvfile) headers = [ 'case id', 'case type', 'creating form id', 'referenced id', 'referenced_type', 'index relationship', 'index identifier', 'owner id', 'owner name', 'opened by id', 'opened by name', ] if debug: headers.append('app version') writer.writerow(headers) for case in cases_with_invalid_references: for index in case['indices']: if index['referenced_id'] in invalid_referenced_ids: form_id = case['xform_ids'][0] row = [ case['_id'], case['type'], form_id, index['referenced_id'], index['referenced_type'], index['relationship'], index['identifier'], case['owner_id'], cached_owner_id_to_display(case['owner_id']), case['opened_by'], cached_owner_id_to_display(case['opened_by']), ] if debug: form = FormAccessors(domain=domain).get_form(form_id) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row.append(app_version_info.build_version) writer.writerow(row) if cleanup: missing = set() deleted = set() exists = set() for invalid_id in invalid_referenced_ids: try: case = CaseAccessors(domain).get_case(invalid_id) except CaseNotFound: missing.add(invalid_id) else: if case.is_deleted: deleted.add(case) else: exists.add(case)
class SubmissionPost(object): def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None, force_logs=False): assert domain, "'domain' is required" assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db if case_db: assert case_db.domain == domain self.force_logs = force_logs self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 self.track_load = form_load_counter("form_submission", domain) def _set_submission_properties(self, xform): # attaches shared properties of the request to the document. # used on forms and errors xform.submit_ip = self.submit_ip xform.path = self.path xform.openrosa_headers = self.openrosa_headers xform.last_sync_token = self.last_sync_token if self.received_on: xform.received_on = self.received_on if self.date_header: xform.date_header = self.date_header xform.app_id = self.app_id xform.build_id = self.build_id xform.export_tag = ["domain", "xmlns"] xform.partial_submission = self.partial_submission return xform def _handle_known_error(self, error, instance, xforms): # errors we know about related to the content of the form # log the error and respond with a success code so that the phone doesn't # keep trying to send the form xforms[0] = _transform_instance_to_error(self.interface, error, instance) # this is usually just one document, but if an edit errored we want # to save the deprecated form as well self.interface.save_processed_models(xforms) def _handle_basic_failure_modes(self): if any_migrations_in_progress(self.domain): # keep submissions on the phone # until ready to start accepting again return HttpResponse(status=503) if not self.auth_context.is_valid(): return HttpResponseForbidden('Bad auth') if isinstance(self.instance, BadRequest): return HttpResponseBadRequest(self.instance.message) def _post_process_form(self, xform): self._set_submission_properties(xform) found_old = scrub_meta(xform) legacy_notification_assert(not found_old, 'Form with old metadata submitted', xform.form_id) def _get_success_message(self, instance, cases=None): ''' Formplayer requests get a detailed success message pointing to the form/case affected. All other requests get a generic message. Message is formatted with markdown. ''' if not instance.metadata or instance.metadata.deviceID != FORMPLAYER_DEVICE_ID: return ' √ ' messages = [] user = CouchUser.get_by_user_id(instance.user_id) if not user or not user.is_web_user(): return _('Form successfully saved!') from corehq.apps.export.views.list import CaseExportListView, FormExportListView from corehq.apps.export.views.utils import can_view_case_exports, can_view_form_exports from corehq.apps.reports.views import CaseDataView, FormDataView form_link = case_link = form_export_link = case_export_link = None form_view = 'corehq.apps.reports.standard.inspect.SubmitHistory' if has_permission_to_view_report(user, instance.domain, form_view): form_link = reverse(FormDataView.urlname, args=[instance.domain, instance.form_id]) case_view = 'corehq.apps.reports.standard.cases.basic.CaseListReport' if cases and has_permission_to_view_report(user, instance.domain, case_view): if len(cases) == 1: case_link = reverse(CaseDataView.urlname, args=[instance.domain, cases[0].case_id]) else: case_link = ", ".join(["[{}]({})".format( c.name, reverse(CaseDataView.urlname, args=[instance.domain, c.case_id]) ) for c in cases]) if can_view_form_exports(user, instance.domain): form_export_link = reverse(FormExportListView.urlname, args=[instance.domain]) if cases and can_view_case_exports(user, instance.domain): case_export_link = reverse(CaseExportListView.urlname, args=[instance.domain]) # Start with generic message messages.append(_('Form successfully saved!')) # Add link to form/case if possible if form_link and case_link: if len(cases) == 1: messages.append( _("You submitted [this form]({}), which affected [this case]({}).") .format(form_link, case_link)) else: messages.append( _("You submitted [this form]({}), which affected these cases: {}.") .format(form_link, case_link)) elif form_link: messages.append(_("You submitted [this form]({}).").format(form_link)) elif case_link: if len(cases) == 1: messages.append(_("Your form affected [this case]({}).").format(case_link)) else: messages.append(_("Your form affected these cases: {}.").format(case_link)) # Add link to all form/case exports if form_export_link and case_export_link: messages.append( _("Click to export your [case]({}) or [form]({}) data.") .format(case_export_link, form_export_link)) elif form_export_link: messages.append(_("Click to export your [form data]({}).").format(form_export_link)) elif case_export_link: messages.append(_("Click to export your [case data]({}).").format(case_export_link)) return "\n\n".join(messages) def run(self): self.track_load() failure_response = self._handle_basic_failure_modes() if failure_response: return FormProcessingResult(failure_response, None, [], [], 'known_failures') result = process_xform_xml(self.domain, self.instance, self.attachments, self.auth_context.to_json()) submitted_form = result.submitted_form self._post_process_form(submitted_form) self._invalidate_caches(submitted_form) if submitted_form.is_submission_error_log: self.formdb.save_new_form(submitted_form) response = None try: xml = self.instance.decode() except UnicodeDecodeError: pass else: if 'log_subreport' in xml: response = self.get_exception_response_and_log( 'Badly formed device log', submitted_form, self.path ) if not response: response = self.get_exception_response_and_log( 'Problem receiving submission', submitted_form, self.path ) return FormProcessingResult(response, None, [], [], 'submission_error_log') if submitted_form.xmlns == SYSTEM_ACTION_XMLNS: return self.handle_system_action(submitted_form) if submitted_form.xmlns == DEVICE_LOG_XMLNS: return self.process_device_log(submitted_form) cases = [] ledgers = [] submission_type = 'unknown' openrosa_kwargs = {} with result.get_locked_forms() as xforms: if len(xforms) > 1: self.track_load(len(xforms) - 1) if self.case_db: case_db_cache = self.case_db case_db_cache.cached_xforms.extend(xforms) else: case_db_cache = self.interface.casedb_cache( domain=self.domain, lock=True, deleted_ok=True, xforms=xforms, load_src="form_submission", ) with case_db_cache as case_db: instance = xforms[0] if instance.is_duplicate: with tracer.trace('submission.process_duplicate'): submission_type = 'duplicate' existing_form = xforms[1] stub = UnfinishedSubmissionStub.objects.filter( domain=instance.domain, xform_id=existing_form.form_id ).first() result = None if stub: from corehq.form_processor.reprocess import reprocess_unfinished_stub_with_form result = reprocess_unfinished_stub_with_form(stub, existing_form, lock=False) elif existing_form.is_error: from corehq.form_processor.reprocess import reprocess_form result = reprocess_form(existing_form, lock_form=False) if result and result.error: submission_type = 'error' openrosa_kwargs['error_message'] = result.error if existing_form.is_error: openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE else: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE else: self.interface.save_processed_models([instance]) elif not instance.is_error: submission_type = 'normal' try: case_stock_result = self.process_xforms_for_cases(xforms, case_db) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: self._handle_known_error(e, instance, xforms) submission_type = 'error' openrosa_kwargs['error_nature'] = ResponseNature.PROCESSING_FAILURE except Exception as e: # handle / log the error and reraise so the phone knows to resubmit # note that in the case of edit submissions this won't flag the previous # submission as having been edited. this is intentional, since we should treat # this use case as if the edit "failed" handle_unexpected_error(self.interface, instance, e) raise else: instance.initial_processing_complete = True openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms, case_stock_result) if openrosa_kwargs['error_message']: openrosa_kwargs['error_nature'] = ResponseNature.POST_PROCESSING_FAILURE cases = case_stock_result.case_models ledgers = case_stock_result.stock_result.models_to_save openrosa_kwargs['success_message'] = self._get_success_message(instance, cases=cases) elif instance.is_error: submission_type = 'error' response = self._get_open_rosa_response(instance, **openrosa_kwargs) return FormProcessingResult(response, instance, cases, ledgers, submission_type) def _conditionally_send_device_logs_to_sumologic(self, instance): url = getattr(settings, 'SUMOLOGIC_URL', None) if url and SUMOLOGIC_LOGS.enabled(instance.form_data.get('device_id'), NAMESPACE_OTHER): SumoLogicLog(self.domain, instance).send_data(url) def _invalidate_caches(self, xform): for device_id in {None, xform.metadata.deviceID if xform.metadata else None}: self._invalidate_restore_payload_path_cache(xform, device_id) if ASYNC_RESTORE.enabled(self.domain): self._invalidate_async_restore_task_id_cache(xform, device_id) def _invalidate_restore_payload_path_cache(self, xform, device_id): """invalidate cached initial restores""" restore_payload_path_cache = RestorePayloadPathCache( domain=self.domain, user_id=xform.user_id, sync_log_id=xform.last_sync_token, device_id=device_id, ) restore_payload_path_cache.invalidate() def _invalidate_async_restore_task_id_cache(self, xform, device_id): async_restore_task_id_cache = AsyncRestoreTaskIdCache( domain=self.domain, user_id=xform.user_id, sync_log_id=self.last_sync_token, device_id=device_id, ) task_id = async_restore_task_id_cache.get_value() if task_id is not None: revoke_celery_task(task_id) async_restore_task_id_cache.invalidate() @tracer.wrap(name='submission.save_models') def save_processed_models(self, case_db, xforms, case_stock_result): instance = xforms[0] try: with unfinished_submission(instance) as unfinished_submission_stub: try: self.interface.save_processed_models( xforms, case_stock_result.case_models, case_stock_result.stock_result ) except PostSaveError: # mark the stub as saved if there's a post save error # but re-raise the error so that the re-processing queue picks it up unfinished_submission_stub.submission_saved() raise else: unfinished_submission_stub.submission_saved() self.do_post_save_actions(case_db, xforms, case_stock_result) except PostSaveError: return "Error performing post save operations" @staticmethod @tracer.wrap(name='submission.post_save_actions') def do_post_save_actions(case_db, xforms, case_stock_result): instance = xforms[0] case_db.clear_changed() try: case_stock_result.case_result.commit_dirtiness_flags() case_stock_result.stock_result.finalize() SubmissionPost._fire_post_save_signals(instance, case_stock_result.case_models) close_extension_cases( case_db, case_stock_result.case_models, "SubmissionPost-%s-close_extensions" % instance.form_id ) except PostSaveError: raise except Exception: notify_exception(get_request(), "Error performing post save actions during form processing", { 'domain': instance.domain, 'form_id': instance.form_id, }) raise PostSaveError @staticmethod @tracer.wrap(name='submission.process_cases_and_stock') def process_xforms_for_cases(xforms, case_db): from casexml.apps.case.xform import process_cases_with_casedb from corehq.apps.commtrack.processing import process_stock instance = xforms[0] case_result = process_cases_with_casedb(xforms, case_db) stock_result = process_stock(xforms, case_db) modified_on_date = instance.received_on if getattr(instance, 'edited_on', None) and instance.edited_on > instance.received_on: modified_on_date = instance.edited_on cases = case_db.get_cases_for_saving(modified_on_date) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, ) def get_response(self): return self.run().response @staticmethod def _fire_post_save_signals(instance, cases): from casexml.apps.case.signals import case_post_save error_message = "Error occurred during form submission post save (%s)" error_details = {'domain': instance.domain, 'form_id': instance.form_id} results = successful_form_received.send_robust(None, xform=instance) has_errors = log_signal_errors(results, error_message, error_details) for case in cases: results = case_post_save.send_robust(case.__class__, case=case) has_errors |= log_signal_errors(results, error_message, error_details) if has_errors: raise PostSaveError def _get_open_rosa_response(self, instance, success_message=None, error_message=None, error_nature=None): if self.is_openrosa_version3: instance_ok = instance.is_normal or instance.is_duplicate has_error = error_message or error_nature if instance_ok and not has_error: response = openrosa_response.get_openarosa_success_response(message=success_message) else: error_message = error_message or instance.problem response = self.get_v3_error_response(error_message, error_nature) else: if instance.is_normal: response = openrosa_response.get_openarosa_success_response() else: response = self.get_v2_submit_error_response(instance) # this hack is required for ODK response["Location"] = self.location # this is a magic thing that we add response['X-CommCareHQ-FormID'] = instance.form_id return response @staticmethod def get_v2_submit_error_response(doc): return OpenRosaResponse( message=doc.problem, nature=ResponseNature.SUBMIT_ERROR, status=201, ).response() @staticmethod def get_v3_error_response(message, nature): """Returns a 422(Unprocessable Entity) response - if nature == 'processing_failure' the mobile device will quarantine this form and not retry it - any other value of `nature` will result in the form being marked as a failure and retrying """ return OpenRosaResponse( message=message, nature=nature, status=422, ).response() @staticmethod def get_exception_response_and_log(msg, error_instance, path): logging.error( msg, extra={ 'submission_path': path, 'form_id': error_instance.form_id, 'error_message': error_instance.problem } ) # This are generally badly formed XML resulting from file corruption, encryption errors # or other errors on the device which can not be recovered from. # To prevent retries of these errors we submit a 422 response with `processing_failure` nature. return OpenRosaResponse( message="There was an error processing the form: %s" % error_instance.problem, nature=ResponseNature.PROCESSING_FAILURE, status=422, ).response() @tracer.wrap(name='submission.handle_system_action') def handle_system_action(self, form): handle_system_action(form, self.auth_context) self.interface.save_processed_models([form]) response = HttpResponse(status=201) return FormProcessingResult(response, form, [], [], 'system-action') @tracer.wrap(name='submission.process_device_log') def process_device_log(self, device_log_form): self._conditionally_send_device_logs_to_sumologic(device_log_form) ignore_device_logs = settings.SERVER_ENVIRONMENT in settings.NO_DEVICE_LOG_ENVS if self.force_logs or not ignore_device_logs: try: process_device_log(self.domain, device_log_form, self.force_logs) except Exception as e: notify_exception(None, "Error processing device log", details={ 'xml': self.instance, 'domain': self.domain }) e.sentry_capture = False raise response = self._get_open_rosa_response(device_log_form) return FormProcessingResult(response, device_log_form, [], [], 'device-log')
def setUp(self): super(TestReprocessDuringSubmission, self).setUp() self.factory = CaseFactory(domain=self.domain) self.formdb = FormAccessors(self.domain) self.casedb = CaseAccessors(self.domain) self.ledgerdb = LedgerAccessors(self.domain)