def handle(self, domain, folder_path, **options): if os.path.exists(folder_path): if not os.path.isdir(folder_path): raise CommandError( 'Folder path must be the path to a directory') else: os.mkdir(folder_path) form_accessors = FormAccessors(domain) form_ids = form_accessors.get_all_form_ids_in_domain() for form in form_accessors.iter_forms(form_ids): form_path = os.path.join(folder_path, form.form_id) if not os.path.exists(form_path): os.mkdir(form_path) form_meta = FormMetadata( user_id=form.user_id, received_on=form.received_on, app_id=form.app_id, build_id=form.build_id, attachments=list(form.attachments.keys()), auth_context=form.auth_context, ) with open(os.path.join(form_path, 'metadata.json'), 'w', encoding='utf-8') as meta: form_meta_data = json.dumps(form_meta.to_json()) if six.PY2: form_meta_data = form_meta_data.decode('utf-8') meta.write(form_meta_data) xml = form.get_xml() with open(os.path.join(form_path, 'form.xml'), 'wb') as f: f.write(xml) for name, meta in form.attachments.items(): with open(os.path.join(form_path, name), 'wb') as f: f.write(form.get_attachment(name))
def update_responses(self, xform, value_responses_map, user_id): """ Update a set of question responses. Returns a list of any questions that were not found in the xform. """ from corehq.form_processor.interfaces.dbaccessors import FormAccessors from corehq.form_processor.utils.xform import update_response errors = [] xml = xform.get_xml_element() for question, response in value_responses_map.items(): try: update_response(xml, question, response, xmlns=xform.xmlns) except XFormQuestionValueNotFound: errors.append(question) existing_form = FormAccessors(xform.domain).get_with_attachments(xform.get_id) existing_form, new_form = self.processor.new_form_from_old(existing_form, xml, value_responses_map, user_id) self.save_processed_models([new_form, existing_form]) return errors
def reprocess_archive_stubs(): # Check for archive stubs from corehq.form_processor.interfaces.dbaccessors import FormAccessors from couchforms.models import UnfinishedArchiveStub stubs = UnfinishedArchiveStub.objects.filter() datadog_gauge('commcare.unfinished_archive_stubs', len(stubs)) start = time.time() cutoff = start + timedelta(minutes=4).total_seconds() for stub in stubs: # Exit this task after 4 minutes so that the same stub isn't ever processed in multiple queues. if time.time() - start > cutoff: return xform = FormAccessors(stub.domain).get_form(form_id=stub.xform_id) # If the history wasn't updated the first time around, run the whole thing again. if not stub.history_updated: if stub.archive: xform.archive(user_id=stub.user_id) else: xform.unarchive(user_id=stub.user_id) # If the history was updated the first time around, just send the update to kafka else: xform.publish_archive_action_to_kafka(user_id=stub.user_id, archive=stub.archive)
def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db
def test_deleted_case_migration(self): parent_case_id = uuid.uuid4().hex child_case_id = uuid.uuid4().hex parent_case = create_and_save_a_case(self.domain_name, case_id=parent_case_id, case_name='test parent') child_case = create_and_save_a_case(self.domain_name, case_id=child_case_id, case_name='test child') set_parent_case(self.domain_name, child_case, parent_case) form_ids = self._get_form_ids() self.assertEqual(3, len(form_ids)) FormAccessors(self.domain.name).soft_delete_forms( form_ids, datetime.utcnow(), 'test-deletion-with-cases') CaseAccessors(self.domain.name).soft_delete_cases( [parent_case_id, child_case_id], datetime.utcnow(), 'test-deletion-with-cases') self.assertEqual( 2, len( get_doc_ids_in_domain_by_type(self.domain_name, "CommCareCase-Deleted", XFormInstance.get_db()))) self._do_migration_and_assert_flags(self.domain_name) self.assertEqual( 2, len( CaseAccessorSQL.get_deleted_case_ids_in_domain( self.domain_name))) self._compare_diffs([]) parent_transactions = CaseAccessorSQL.get_transactions(parent_case_id) self.assertEqual(2, len(parent_transactions)) self.assertTrue(parent_transactions[0].is_case_create) self.assertTrue(parent_transactions[1].is_form_transaction) child_transactions = CaseAccessorSQL.get_transactions(child_case_id) self.assertEqual(2, len(child_transactions)) self.assertTrue(child_transactions[0].is_case_create) self.assertTrue(child_transactions[1].is_case_index)
def testSubmitDuplicate(self): file, res = self._submit('simple_form.xml') self.assertEqual(201, res.status_code) self.assertIn(" √ ".encode('utf-8'), res.content) file, res = self._submit('simple_form.xml') self.assertEqual(201, res.status_code) _, res_openrosa3 = self._submit('simple_form.xml', open_rosa_header=OPENROSA_VERSION_3) self.assertEqual(201, res_openrosa3.status_code) self.assertIn("Form is a duplicate", res.content.decode('utf-8')) # make sure we logged it [log] = FormAccessors(self.domain.name).get_forms_by_type( 'XFormDuplicate', limit=1) self.assertIsNotNone(log) self.assertIn("Form is a duplicate", log.problem) with open(file, 'rb') as f: self.assertEqual(f.read(), log.get_xml())
def _test_submission_error_post_save(self, openrosa_version): evil_laugh = "mwa ha ha!" with failing_signal_handler(evil_laugh): file, res = self._submit("simple_form.xml", openrosa_version) if openrosa_version == OPENROSA_VERSION_3: self.assertEqual(422, res.status_code) self.assertIn( ResponseNature.POST_PROCESSING_FAILURE.encode('utf-8'), res.content) else: self.assertEqual(201, res.status_code) self.assertIn(ResponseNature.SUBMIT_SUCCESS.encode('utf-8'), res.content) form_id = 'ad38211be256653bceac8e2156475664' form = FormAccessors(self.domain.name).get_form(form_id) self.assertTrue(form.is_normal) self.assertTrue(form.initial_processing_complete) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, xform_id=form_id, saved=True).all() self.assertEqual(1, len(stubs))
def test_archive_against_deleted_case(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, p1='p1', form_extras={'received_on': now}) _post_util(case_id=case_id, p2='p2', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p3='p3', form_extras={'received_on': now + timedelta(seconds=2)}) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) case_accessors.soft_delete_cases([case_id]) [f1, f2, f3] = case.xform_ids f2_doc = FormAccessors(REBUILD_TEST_DOMAIN).get_form(f2) f2_doc.archive() case = case_accessors.get_case(case_id) self.assertTrue(case.is_deleted)
def set_up_form(cls): cls.form_id = uuid4().hex user_id = uuid4().hex cls.case_update = { 'year': '1970', 'breakfast': 'spam egg spam spam bacon spam', 'price': '2.40', 'album_release': '1972-09-08', 'breakfast_oclock': '09:00:00', 'breakfast_exactly': '1972-09-08T09:00:00.000Z', } builder = FormSubmissionBuilder( form_id=cls.form_id, form_properties={ 'name': 'spam', **cls.case_update, }, case_blocks=[ CaseBlock(case_id=uuid4().hex, create=True, case_type='sketch', case_name='spam', owner_id=user_id, update=cls.case_update) ], metadata=TestFormMetadata( domain=cls.domain, user_id=user_id, ), ) submit_form_locally(builder.as_xml_string(), cls.domain) cls.form = FormAccessors(cls.domain).get_form(cls.form_id) form_json_gen = FormRepeaterJsonPayloadGenerator(None) cls.form_json_payload_info = cls.get_payload_info(form_json_gen) form_dict_gen = FormDictPayloadGenerator(None) cls.form_dict_payload_info = cls.get_payload_info(form_dict_gen)
def test_archie_modified_on(self): case_id = uuid.uuid4().hex now = datetime.utcnow().replace(microsecond=0) earlier = now - timedelta(hours=1) way_earlier = now - timedelta(days=1) # make sure we timestamp everything so they have the right order create_block = CaseBlock(case_id, create=True, date_modified=way_earlier) post_case_blocks( [create_block.as_xml()], form_extras={'received_on': way_earlier} ) update_block = CaseBlock(case_id, update={'foo': 'bar'}, date_modified=earlier) post_case_blocks( [update_block.as_xml()], form_extras={'received_on': earlier} ) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) self.assertEqual(earlier, case.modified_on) second_form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(case.xform_ids[-1]) second_form.archive() case = case_accessors.get_case(case_id) self.assertEqual(way_earlier, case.modified_on)
def get_simple_wrapped_form(form_id, metadata=None, save=True, simple_form=SIMPLE_FORM): from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.interfaces.dbaccessors import FormAccessors metadata = metadata or TestFormMetadata() xml = get_simple_form_xml(form_id=form_id, metadata=metadata, simple_form=simple_form) form_json = convert_xform_to_json(xml) interface = FormProcessorInterface(domain=metadata.domain) wrapped_form = interface.new_xform(form_json) wrapped_form.domain = metadata.domain wrapped_form.received_on = metadata.received_on interface.store_attachments(wrapped_form, [Attachment('form.xml', xml, 'text/xml')]) if save: interface.save_processed_models([wrapped_form]) wrapped_form = FormAccessors(metadata.domain).get_form( wrapped_form.form_id) return wrapped_form
def handle(self, domains, file_name, **options): blob_db = get_blob_db() with open(file_name, 'w', encoding='utf-8') as csv_file: field_names = ['domain', 'archived', 'form_id', 'received_on'] csv_writer = csv.DictWriter(csv_file, field_names) csv_writer.writeheader() for domain in domains: self.stdout.write("Handling domain %s" % domain) form_db = FormAccessors(domain) form_ids = form_db.get_all_form_ids_in_domain() form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived')) for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)): if isinstance(form, CouchForm): meta = form.blobs.get(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) elif isinstance(form, XFormInstanceSQL): meta = form.get_attachment_meta(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) else: raise Exception("not sure how we got here")
def setUpClass(cls): super(KafkaPublishingTest, cls).setUpClass() cls.form_accessors = FormAccessors(domain=cls.domain) cls.processor = TestProcessor() cls.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], client_id='test-kafka-form-feed'), processor=cls.processor) cls.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], client_id='test-kafka-case-feed'), processor=cls.processor) cls.process_form_changes = process_pillow_changes( 'DefaultChangeFeedPillow') cls.process_form_changes.add_pillow(cls.form_pillow) cls.process_case_changes = process_pillow_changes( 'DefaultChangeFeedPillow') cls.process_case_changes.add_pillow(cls.case_pillow)
def setUp(self): super(KafkaPublishingTest, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.form_accessors = FormAccessors(domain=self.domain) self.processor = TestProcessor() self.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'), processor=self.processor) self.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'), processor=self.processor) self.ledger_pillow = ConstructedPillow( name='test-kafka-ledger-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'), processor=self.processor)
def handle(self, username, domain, **options): this_form_accessor = FormAccessors(domain=domain) user = CouchUser.get_by_username(username) if not user: logger.info("User {} not found.".format(username)) sys.exit(1) user_id = user._id form_ids = this_form_accessor.get_form_ids_for_user(user_id) input_response = six.moves.input( "Update {} form(s) for user {} in domain {}? (y/n): ".format( len(form_ids), username, domain)) if input_response == "y": for form_data in this_form_accessor.iter_forms(form_ids): form_attachment_xml_new = self.update_form_data( form_data, NEW_USERNAME) this_form_accessor.modify_attachment_xml_and_metadata( form_data, form_attachment_xml_new, NEW_USERNAME) logging.info("Updated {} form(s) for user {} in domain {}".format( len(form_ids), username, domain)) elif input_response == "n": logging.info("No forms updated, exiting.") else: logging.info("Command not recognized. Exiting.")
def perform_resave_on_xforms(domain, start_date, end_date, no_input): _, _, xform_ids_missing_in_es, _ = compare_xforms(domain, 'XFormInstance', start_date, end_date) print("%s Ids found for xforms missing in ES." % len(xform_ids_missing_in_es)) if len(xform_ids_missing_in_es) < 1000: print(xform_ids_missing_in_es) if no_input is not True: ok = input("Type 'ok' to continue: ") if ok != "ok": print("No changes made") return form_accessor = FormAccessors(domain) for xform_ids in chunked(with_progress_bar(xform_ids_missing_in_es), 100): xforms = form_accessor.get_forms(list(xform_ids)) found_xform_ids = set() for xform in xforms: resave_form(domain, xform) found_xform_ids.add(xform.form_id) for xform_id in set(xform_ids) - found_xform_ids: print("form not found %s" % xform_id)
def _test_post(self, file_path, authtype=None, client=None, expected_status=201, expected_auth_context=None, submit_mode=None, expected_response=None): if not client: client = django_digest.test.Client() def _make_url(): url = self.url url_params = {} if authtype: url_params['authtype'] = authtype if submit_mode: url_params['submit_mode'] = submit_mode url = '%s?%s' % (url, urlencode(url_params)) return url url = _make_url() with open(file_path, "r", encoding='utf-8') as f: fileobj = FakeFile( f.read().format( userID=self.user.user_id, instanceID=uuid.uuid4().hex, case_id=uuid.uuid4().hex, ), name=file_path, ) response = client.post(url, {"xml_submission_file": fileobj}) self.assertEqual(response.status_code, expected_status) if expected_response: self.assertEqual(response.content, expected_response) if expected_auth_context is not None: xform_id = response['X-CommCareHQ-FormID'] xform = FormAccessors(self.domain).get_form(xform_id) self.assertEqual(xform.auth_context, expected_auth_context) return xform
def test_patch_case_with_deleted_form_and_unexpected_diff(self): self.submit_form(make_test_form("form-1", case_id="case-1")) case = CaseAccessorCouch.get_case("case-1") case.user_id = "unexpected" case.save() FormAccessors(self.domain_name).soft_delete_forms(["form-1"], datetime.utcnow(), 'test-deletion') self.do_migration(diffs=IGNORE) self.compare_diffs(changes=[ Diff('case-1', 'missing', ['*'], old='*', new=MISSING, reason="deleted forms"), ]) # first patch results in unexpected diff self.do_case_patch() self.compare_diffs(diffs=[ Diff('case-1', 'diff', ['opened_by'], old='3fae4ea4af440efaa53441b5', new='unexpected'), Diff('case-1', 'set_mismatch', path=['xform_ids', '[*]'], old='form-1', new=ANY), ]) self.assert_patched_cases(["case-1"]) # second patch resolves unexpected diff self.do_case_patch() self.compare_diffs() self.assert_backend("sql") self.assertFalse(self._get_case("case-1").deleted)
def test_archive_only_form(self): # check no data in stock states ledger_accessors = LedgerAccessors(self.domain.name) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(0, len(ledger_values)) initial_amounts = [(p._id, float(100)) for p in self.products] form_id = self.submit_xml_form(balance_submission(initial_amounts)) # check that we made stuff def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: self.assertEqual(100, int(state.stock_on_hand)) _assert_initial_state() # archive and confirm commtrack data is cleared form = FormAccessors(self.domain.name).get_form(form_id) form.archive() self.assertEqual(0, len(ledger_accessors.get_ledger_values_for_case(self.sp.case_id))) if should_use_sql_backend(self.domain): self.assertEqual(0, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(0, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(0, StockTransaction.objects.filter(report__form_id=form_id).count()) # unarchive and confirm commtrack data is restored form.unarchive() _assert_initial_state()
def testSubmissionError(self): evil_laugh = "mwa ha ha!" def fail(sender, xform, **kwargs): raise Exception(evil_laugh) successful_form_received.connect(fail) try: file, res = self._submit("simple_form.xml") self.assertEqual(201, res.status_code) self.assertIn(evil_laugh, res.content) # make sure we logged it [log] = FormAccessors(self.domain.name).get_forms_by_type( 'XFormError', limit=1) self.assertIsNotNone(log) self.assertIn(evil_laugh, log.problem) with open(file) as f: self.assertEqual(f.read(), log.get_xml()) finally: successful_form_received.disconnect(fail)
def handle(self, domain, **options): verbose = options["verbose"] or options["dryrun"] succeeded = [] failed = [] error_messages = defaultdict(lambda: 0) problem_ids = self._get_form_ids(domain) prefix = "Processing: " form_iterator = FormAccessors(domain).iter_forms(problem_ids) if not verbose: form_iterator = with_progress_bar(form_iterator, len(problem_ids), prefix=prefix, oneline=False) for form in form_iterator: if verbose: print("%s\t%s\t%s\t%s" % (form.form_id, form.received_on, form.xmlns, form.problem.strip())) if not options["dryrun"]: try: reprocess_xform_error(form) except Exception as e: raise failed.append(form.form_id) error_messages[str(e)] += 1 else: succeeded.append(form.form_id) if not options["dryrun"]: print("%s / %s forms successfully processed, %s failures" % (len(succeeded), len(succeeded) + len(failed), len(failed))) if error_messages: print("The following errors were seen: \n%s" % ("\n".join("%s: %s" % (v, k) for k, v in error_messages.items())))
def reprocess_archive_stubs(): # Check for archive stubs from corehq.form_processor.interfaces.dbaccessors import FormAccessors from couchforms.models import UnfinishedArchiveStub stubs = UnfinishedArchiveStub.objects.filter(attempts__lt=3) metrics_gauge('commcare.unfinished_archive_stubs', len(stubs)) start = time.time() cutoff = start + timedelta(minutes=4).total_seconds() for stub in stubs: # Exit this task after 4 minutes so that tasks remain short if time.time() > cutoff: return try: xform = FormAccessors(stub.domain).get_form(form_id=stub.xform_id) # If the history wasn't updated the first time around, run the whole thing again. if not stub.history_updated: FormAccessors.do_archive(xform, stub.archive, stub.user_id, trigger_signals=True) # If the history was updated the first time around, just send the update to kafka else: FormAccessors.publish_archive_action_to_kafka(xform, stub.user_id, stub.archive) except Exception: # Errors should not prevent processing other stubs notify_exception(None, "Error processing UnfinishedArchiveStub")
def bulk_undo(self, progress_bar=False): chunk_size = 100 result = { 'processed': 0, 'skipped': 0, 'archived': 0, } form_ids = list(self.get_submission_queryset().values_list('form_id', flat=True)) form_id_chunks = chunked(form_ids, chunk_size) if progress_bar: length = len(form_ids) / chunk_size if len(form_ids) % chunk_size > 0: length += 1 form_id_chunks = with_progress_bar(form_id_chunks, length=length) for form_id_chunk in form_id_chunks: archived_form_ids = [] for form in FormAccessors(self.domain).iter_forms(form_id_chunk): result['processed'] += 1 if not form.is_normal or any( [u.creates_case() for u in get_case_updates(form)]): result['skipped'] += 1 continue if not form.is_archived: form.archive(user_id=SYSTEM_USER_ID) result['archived'] += 1 archived_form_ids.append(form.form_id) CaseRuleSubmission.objects.filter( form_id__in=archived_form_ids).update(archived=True) return result
def handle_unexpected_error(interface, instance, exception, message=None): instance = _transform_instance_to_error(interface, exception, instance) _notify_submission_error(instance, exception, instance.problem) FormAccessors(interface.domain).save_new_form(instance)
def payload_doc(self, repeat_record): return FormAccessors(repeat_record.domain).get_form( repeat_record.payload_id)
def undelete_system_forms(domain, deleted_forms, deleted_cases): """The reverse of tag_system_forms_as_deleted; called on user.unretire()""" to_undelete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=False) FormAccessors(domain).soft_undelete_forms(to_undelete)
def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date): to_delete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=True) FormAccessors(domain).soft_delete_forms(to_delete, deletion_date, deletion_id)
def _is_safe_to_modify(form): if form.domain != domain: return False case_ids = get_case_ids_from_form(form) # all cases touched by the form and not already modified for case in CaseAccessors(domain).iter_cases(case_ids - modified_cases): if case.is_deleted != is_deletion: # we can't delete/undelete this form - this would change the state of `case` return False # all cases touched by this form are deleted return True if is_deletion or Domain.get_by_name(domain).use_sql_backend: all_forms = FormAccessors(domain).iter_forms(form_ids_to_modify) else: # accessor.iter_forms doesn't include deleted forms on the couch backend all_forms = list(map(FormAccessors(domain).get_form, form_ids_to_modify)) return [form.form_id for form in all_forms if _is_safe_to_modify(form)] @task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True) def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date): to_delete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=True) FormAccessors(domain).soft_delete_forms(to_delete, deletion_date, deletion_id) @task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True) def undelete_system_forms(domain, deleted_forms, deleted_cases): """The reverse of tag_system_forms_as_deleted; called on user.unretire()"""
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments :returns: A two-tuple: `(<new form>, <duplicate form or None>)` The new form may have a different `form_id` than `new_doc.form_id`. """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id try: existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id) except ResourceNotFound: # Original form processing failed but left behind a form doc with no # attachments. It's safe to delete this now since we're going to re-process # the form anyway. from couchforms.models import XFormInstance XFormInstance.get_db().delete_doc(conflict_id) return new_doc, None try: existing_md5 = existing_doc.xml_md5() except MissingFormXml: existing_md5 = None if not existing_doc.is_error: existing_doc.problem = 'Missing form.xml' new_md5 = new_doc.xml_md5() if existing_md5 != new_md5: _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False) if new_doc.xmlns != existing_doc.xmlns: # if the XMLNS has changed this probably isn't a form edit # it could be a UUID clash (yes we've had that before) # Assign a new ID to the form and process as normal + notify_admins xform = interface.assign_new_id(new_doc) _soft_assert( False, "Potential UUID clash", { 'incoming_form_id': conflict_id, 'existing_form_id': existing_doc.form_id, 'new_form_id': xform.form_id, 'incoming_xmlns': new_doc.xmlns, 'existing_xmlns': existing_doc.xmlns, 'domain': new_doc.domain, } ) return xform, None else: if existing_doc.is_error and not existing_doc.initial_processing_complete: # edge case from ICDS where a form errors and then future re-submissions of the same # form do not have the same MD5 hash due to a bug on mobile: # see https://dimagi-dev.atlassian.net/browse/ICDS-376 # since we have a new form and the old one was not successfully processed # we can effectively ignore this form and process the new one as normal if not interface.use_sql_domain: new_doc._rev, existing_doc._rev = existing_doc._rev, new_doc._rev interface.assign_new_id(existing_doc) existing_doc.save() return new_doc, None else: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID NotAllowed.check(new_doc.domain) existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return new_doc, existing_doc else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return duplicate, existing_doc
def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain')