def handle(self, domain, folder_path, **options):
if os.path.exists(folder_path):
if not os.path.isdir(folder_path):
raise CommandError(
'Folder path must be the path to a directory')
else:
os.mkdir(folder_path)
form_accessors = FormAccessors(domain)
form_ids = form_accessors.get_all_form_ids_in_domain()
for form in form_accessors.iter_forms(form_ids):
form_path = os.path.join(folder_path, form.form_id)
if not os.path.exists(form_path):
os.mkdir(form_path)
form_meta = FormMetadata(
user_id=form.user_id,
received_on=form.received_on,
app_id=form.app_id,
build_id=form.build_id,
attachments=list(form.attachments.keys()),
auth_context=form.auth_context,
)
with open(os.path.join(form_path, 'metadata.json'),
'w',
encoding='utf-8') as meta:
form_meta_data = json.dumps(form_meta.to_json())
if six.PY2:
form_meta_data = form_meta_data.decode('utf-8')
meta.write(form_meta_data)
xml = form.get_xml()
with open(os.path.join(form_path, 'form.xml'), 'wb') as f:
f.write(xml)
for name, meta in form.attachments.items():
with open(os.path.join(form_path, name), 'wb') as f:
f.write(form.get_attachment(name))
def update_responses(self, xform, value_responses_map, user_id):
"""
Update a set of question responses. Returns a list of any
questions that were not found in the xform.
"""
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.utils.xform import update_response
errors = []
xml = xform.get_xml_element()
for question, response in value_responses_map.items():
try:
update_response(xml, question, response, xmlns=xform.xmlns)
except XFormQuestionValueNotFound:
errors.append(question)
existing_form = FormAccessors(xform.domain).get_with_attachments(xform.get_id)
existing_form, new_form = self.processor.new_form_from_old(existing_form, xml,
value_responses_map, user_id)
self.save_processed_models([new_form, existing_form])
return errors
def reprocess_archive_stubs():
# Check for archive stubs
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from couchforms.models import UnfinishedArchiveStub
stubs = UnfinishedArchiveStub.objects.filter()
datadog_gauge('commcare.unfinished_archive_stubs', len(stubs))
start = time.time()
cutoff = start + timedelta(minutes=4).total_seconds()
for stub in stubs:
# Exit this task after 4 minutes so that the same stub isn't ever processed in multiple queues.
if time.time() - start > cutoff:
return
xform = FormAccessors(stub.domain).get_form(form_id=stub.xform_id)
# If the history wasn't updated the first time around, run the whole thing again.
if not stub.history_updated:
if stub.archive:
xform.archive(user_id=stub.user_id)
else:
xform.unarchive(user_id=stub.user_id)
# If the history was updated the first time around, just send the update to kafka
else:
xform.publish_archive_action_to_kafka(user_id=stub.user_id, archive=stub.archive)
def __init__(self,
instance=None,
attachments=None,
auth_context=None,
domain=None,
app_id=None,
build_id=None,
path=None,
location=None,
submit_ip=None,
openrosa_headers=None,
last_sync_token=None,
received_on=None,
date_header=None,
partial_submission=False,
case_db=None):
assert domain, domain
assert instance, instance
assert not isinstance(instance, HttpRequest), instance
self.domain = domain
self.app_id = app_id
self.build_id = build_id
# get_location has good default
self.location = location or couchforms.get_location()
self.received_on = received_on
self.date_header = date_header
self.submit_ip = submit_ip
self.last_sync_token = last_sync_token
self.openrosa_headers = openrosa_headers or {}
self.instance = instance
self.attachments = attachments or {}
self.auth_context = auth_context or DefaultAuthContext()
self.path = path
self.interface = FormProcessorInterface(domain)
self.formdb = FormAccessors(domain)
self.partial_submission = partial_submission
# always None except in the case where a system form is being processed as part of another submission
# e.g. for closing extension cases
self.case_db = case_db
def test_deleted_case_migration(self):
parent_case_id = uuid.uuid4().hex
child_case_id = uuid.uuid4().hex
parent_case = create_and_save_a_case(self.domain_name,
case_id=parent_case_id,
case_name='test parent')
child_case = create_and_save_a_case(self.domain_name,
case_id=child_case_id,
case_name='test child')
set_parent_case(self.domain_name, child_case, parent_case)
form_ids = self._get_form_ids()
self.assertEqual(3, len(form_ids))
FormAccessors(self.domain.name).soft_delete_forms(
form_ids, datetime.utcnow(), 'test-deletion-with-cases')
CaseAccessors(self.domain.name).soft_delete_cases(
[parent_case_id, child_case_id], datetime.utcnow(),
'test-deletion-with-cases')
self.assertEqual(
2,
len(
get_doc_ids_in_domain_by_type(self.domain_name,
"CommCareCase-Deleted",
XFormInstance.get_db())))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(
2,
len(
CaseAccessorSQL.get_deleted_case_ids_in_domain(
self.domain_name)))
self._compare_diffs([])
parent_transactions = CaseAccessorSQL.get_transactions(parent_case_id)
self.assertEqual(2, len(parent_transactions))
self.assertTrue(parent_transactions[0].is_case_create)
self.assertTrue(parent_transactions[1].is_form_transaction)
child_transactions = CaseAccessorSQL.get_transactions(child_case_id)
self.assertEqual(2, len(child_transactions))
self.assertTrue(child_transactions[0].is_case_create)
self.assertTrue(child_transactions[1].is_case_index)
def testSubmitDuplicate(self):
file, res = self._submit('simple_form.xml')
self.assertEqual(201, res.status_code)
self.assertIn(" √ ".encode('utf-8'), res.content)
file, res = self._submit('simple_form.xml')
self.assertEqual(201, res.status_code)
_, res_openrosa3 = self._submit('simple_form.xml',
open_rosa_header=OPENROSA_VERSION_3)
self.assertEqual(201, res_openrosa3.status_code)
self.assertIn("Form is a duplicate", res.content.decode('utf-8'))
# make sure we logged it
[log] = FormAccessors(self.domain.name).get_forms_by_type(
'XFormDuplicate', limit=1)
self.assertIsNotNone(log)
self.assertIn("Form is a duplicate", log.problem)
with open(file, 'rb') as f:
self.assertEqual(f.read(), log.get_xml())
def _test_submission_error_post_save(self, openrosa_version):
evil_laugh = "mwa ha ha!"
with failing_signal_handler(evil_laugh):
file, res = self._submit("simple_form.xml", openrosa_version)
if openrosa_version == OPENROSA_VERSION_3:
self.assertEqual(422, res.status_code)
self.assertIn(
ResponseNature.POST_PROCESSING_FAILURE.encode('utf-8'),
res.content)
else:
self.assertEqual(201, res.status_code)
self.assertIn(ResponseNature.SUBMIT_SUCCESS.encode('utf-8'),
res.content)
form_id = 'ad38211be256653bceac8e2156475664'
form = FormAccessors(self.domain.name).get_form(form_id)
self.assertTrue(form.is_normal)
self.assertTrue(form.initial_processing_complete)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
xform_id=form_id,
saved=True).all()
self.assertEqual(1, len(stubs))
def test_archive_against_deleted_case(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True,
p1='p1',
form_extras={'received_on': now})
_post_util(case_id=case_id,
p2='p2',
form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id,
p3='p3',
form_extras={'received_on': now + timedelta(seconds=2)})
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
case_accessors.soft_delete_cases([case_id])
[f1, f2, f3] = case.xform_ids
f2_doc = FormAccessors(REBUILD_TEST_DOMAIN).get_form(f2)
f2_doc.archive()
case = case_accessors.get_case(case_id)
self.assertTrue(case.is_deleted)
def set_up_form(cls):
cls.form_id = uuid4().hex
user_id = uuid4().hex
cls.case_update = {
'year': '1970',
'breakfast': 'spam egg spam spam bacon spam',
'price': '2.40',
'album_release': '1972-09-08',
'breakfast_oclock': '09:00:00',
'breakfast_exactly': '1972-09-08T09:00:00.000Z',
}
builder = FormSubmissionBuilder(
form_id=cls.form_id,
form_properties={
'name': 'spam',
**cls.case_update,
},
case_blocks=[
CaseBlock(case_id=uuid4().hex,
create=True,
case_type='sketch',
case_name='spam',
owner_id=user_id,
update=cls.case_update)
],
metadata=TestFormMetadata(
domain=cls.domain,
user_id=user_id,
),
)
submit_form_locally(builder.as_xml_string(), cls.domain)
cls.form = FormAccessors(cls.domain).get_form(cls.form_id)
form_json_gen = FormRepeaterJsonPayloadGenerator(None)
cls.form_json_payload_info = cls.get_payload_info(form_json_gen)
form_dict_gen = FormDictPayloadGenerator(None)
cls.form_dict_payload_info = cls.get_payload_info(form_dict_gen)
def test_archie_modified_on(self):
case_id = uuid.uuid4().hex
now = datetime.utcnow().replace(microsecond=0)
earlier = now - timedelta(hours=1)
way_earlier = now - timedelta(days=1)
# make sure we timestamp everything so they have the right order
create_block = CaseBlock(case_id, create=True, date_modified=way_earlier)
post_case_blocks(
[create_block.as_xml()], form_extras={'received_on': way_earlier}
)
update_block = CaseBlock(case_id, update={'foo': 'bar'}, date_modified=earlier)
post_case_blocks(
[update_block.as_xml()], form_extras={'received_on': earlier}
)
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
self.assertEqual(earlier, case.modified_on)
second_form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(case.xform_ids[-1])
second_form.archive()
case = case_accessors.get_case(case_id)
self.assertEqual(way_earlier, case.modified_on)
def get_simple_wrapped_form(form_id,
metadata=None,
save=True,
simple_form=SIMPLE_FORM):
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
metadata = metadata or TestFormMetadata()
xml = get_simple_form_xml(form_id=form_id,
metadata=metadata,
simple_form=simple_form)
form_json = convert_xform_to_json(xml)
interface = FormProcessorInterface(domain=metadata.domain)
wrapped_form = interface.new_xform(form_json)
wrapped_form.domain = metadata.domain
wrapped_form.received_on = metadata.received_on
interface.store_attachments(wrapped_form,
[Attachment('form.xml', xml, 'text/xml')])
if save:
interface.save_processed_models([wrapped_form])
wrapped_form = FormAccessors(metadata.domain).get_form(
wrapped_form.form_id)
return wrapped_form
def handle(self, domains, file_name, **options):
blob_db = get_blob_db()
with open(file_name, 'w', encoding='utf-8') as csv_file:
field_names = ['domain', 'archived', 'form_id', 'received_on']
csv_writer = csv.DictWriter(csv_file, field_names)
csv_writer.writeheader()
for domain in domains:
self.stdout.write("Handling domain %s" % domain)
form_db = FormAccessors(domain)
form_ids = form_db.get_all_form_ids_in_domain()
form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived'))
for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)):
if isinstance(form, CouchForm):
meta = form.blobs.get(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
elif isinstance(form, XFormInstanceSQL):
meta = form.get_attachment_meta(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
else:
raise Exception("not sure how we got here")
def setUpClass(cls):
super(KafkaPublishingTest, cls).setUpClass()
cls.form_accessors = FormAccessors(domain=cls.domain)
cls.processor = TestProcessor()
cls.form_pillow = ConstructedPillow(
name='test-kafka-form-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL],
client_id='test-kafka-form-feed'),
processor=cls.processor)
cls.case_pillow = ConstructedPillow(
name='test-kafka-case-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL],
client_id='test-kafka-case-feed'),
processor=cls.processor)
cls.process_form_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_form_changes.add_pillow(cls.form_pillow)
cls.process_case_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_case_changes.add_pillow(cls.case_pillow)
def setUp(self):
super(KafkaPublishingTest, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.form_accessors = FormAccessors(domain=self.domain)
self.processor = TestProcessor()
self.form_pillow = ConstructedPillow(
name='test-kafka-form-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL],
group_id='test-kafka-form-feed'),
processor=self.processor)
self.case_pillow = ConstructedPillow(
name='test-kafka-case-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL],
group_id='test-kafka-case-feed'),
processor=self.processor)
self.ledger_pillow = ConstructedPillow(
name='test-kafka-ledger-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.LEDGER],
group_id='test-kafka-ledger-feed'),
processor=self.processor)
def handle(self, username, domain, **options):
this_form_accessor = FormAccessors(domain=domain)
user = CouchUser.get_by_username(username)
if not user:
logger.info("User {} not found.".format(username))
sys.exit(1)
user_id = user._id
form_ids = this_form_accessor.get_form_ids_for_user(user_id)
input_response = six.moves.input(
"Update {} form(s) for user {} in domain {}? (y/n): ".format(
len(form_ids), username, domain))
if input_response == "y":
for form_data in this_form_accessor.iter_forms(form_ids):
form_attachment_xml_new = self.update_form_data(
form_data, NEW_USERNAME)
this_form_accessor.modify_attachment_xml_and_metadata(
form_data, form_attachment_xml_new, NEW_USERNAME)
logging.info("Updated {} form(s) for user {} in domain {}".format(
len(form_ids), username, domain))
elif input_response == "n":
logging.info("No forms updated, exiting.")
else:
logging.info("Command not recognized. Exiting.")
def perform_resave_on_xforms(domain, start_date, end_date, no_input):
_, _, xform_ids_missing_in_es, _ = compare_xforms(domain, 'XFormInstance',
start_date, end_date)
print("%s Ids found for xforms missing in ES." %
len(xform_ids_missing_in_es))
if len(xform_ids_missing_in_es) < 1000:
print(xform_ids_missing_in_es)
if no_input is not True:
ok = input("Type 'ok' to continue: ")
if ok != "ok":
print("No changes made")
return
form_accessor = FormAccessors(domain)
for xform_ids in chunked(with_progress_bar(xform_ids_missing_in_es), 100):
xforms = form_accessor.get_forms(list(xform_ids))
found_xform_ids = set()
for xform in xforms:
resave_form(domain, xform)
found_xform_ids.add(xform.form_id)
for xform_id in set(xform_ids) - found_xform_ids:
print("form not found %s" % xform_id)
def _test_post(self, file_path, authtype=None, client=None,
expected_status=201, expected_auth_context=None,
submit_mode=None, expected_response=None):
if not client:
client = django_digest.test.Client()
def _make_url():
url = self.url
url_params = {}
if authtype:
url_params['authtype'] = authtype
if submit_mode:
url_params['submit_mode'] = submit_mode
url = '%s?%s' % (url, urlencode(url_params))
return url
url = _make_url()
with open(file_path, "r", encoding='utf-8') as f:
fileobj = FakeFile(
f.read().format(
userID=self.user.user_id,
instanceID=uuid.uuid4().hex,
case_id=uuid.uuid4().hex,
),
name=file_path,
)
response = client.post(url, {"xml_submission_file": fileobj})
self.assertEqual(response.status_code, expected_status)
if expected_response:
self.assertEqual(response.content, expected_response)
if expected_auth_context is not None:
xform_id = response['X-CommCareHQ-FormID']
xform = FormAccessors(self.domain).get_form(xform_id)
self.assertEqual(xform.auth_context, expected_auth_context)
return xform
def test_patch_case_with_deleted_form_and_unexpected_diff(self):
self.submit_form(make_test_form("form-1", case_id="case-1"))
case = CaseAccessorCouch.get_case("case-1")
case.user_id = "unexpected"
case.save()
FormAccessors(self.domain_name).soft_delete_forms(["form-1"],
datetime.utcnow(),
'test-deletion')
self.do_migration(diffs=IGNORE)
self.compare_diffs(changes=[
Diff('case-1',
'missing', ['*'],
old='*',
new=MISSING,
reason="deleted forms"),
])
# first patch results in unexpected diff
self.do_case_patch()
self.compare_diffs(diffs=[
Diff('case-1',
'diff', ['opened_by'],
old='3fae4ea4af440efaa53441b5',
new='unexpected'),
Diff('case-1',
'set_mismatch',
path=['xform_ids', '[*]'],
old='form-1',
new=ANY),
])
self.assert_patched_cases(["case-1"])
# second patch resolves unexpected diff
self.do_case_patch()
self.compare_diffs()
self.assert_backend("sql")
self.assertFalse(self._get_case("case-1").deleted)
def test_archive_only_form(self):
# check no data in stock states
ledger_accessors = LedgerAccessors(self.domain.name)
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(0, len(ledger_values))
initial_amounts = [(p._id, float(100)) for p in self.products]
form_id = self.submit_xml_form(balance_submission(initial_amounts))
# check that we made stuff
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
self.assertEqual(100, int(state.stock_on_hand))
_assert_initial_state()
# archive and confirm commtrack data is cleared
form = FormAccessors(self.domain.name).get_form(form_id)
form.archive()
self.assertEqual(0, len(ledger_accessors.get_ledger_values_for_case(self.sp.case_id)))
if should_use_sql_backend(self.domain):
self.assertEqual(0, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(0, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=form_id).count())
# unarchive and confirm commtrack data is restored
form.unarchive()
_assert_initial_state()
def testSubmissionError(self):
evil_laugh = "mwa ha ha!"
def fail(sender, xform, **kwargs):
raise Exception(evil_laugh)
successful_form_received.connect(fail)
try:
file, res = self._submit("simple_form.xml")
self.assertEqual(201, res.status_code)
self.assertIn(evil_laugh, res.content)
# make sure we logged it
[log] = FormAccessors(self.domain.name).get_forms_by_type(
'XFormError', limit=1)
self.assertIsNotNone(log)
self.assertIn(evil_laugh, log.problem)
with open(file) as f:
self.assertEqual(f.read(), log.get_xml())
finally:
successful_form_received.disconnect(fail)
def handle(self, domain, **options):
verbose = options["verbose"] or options["dryrun"]
succeeded = []
failed = []
error_messages = defaultdict(lambda: 0)
problem_ids = self._get_form_ids(domain)
prefix = "Processing: "
form_iterator = FormAccessors(domain).iter_forms(problem_ids)
if not verbose:
form_iterator = with_progress_bar(form_iterator,
len(problem_ids),
prefix=prefix,
oneline=False)
for form in form_iterator:
if verbose:
print("%s\t%s\t%s\t%s" % (form.form_id, form.received_on,
form.xmlns, form.problem.strip()))
if not options["dryrun"]:
try:
reprocess_xform_error(form)
except Exception as e:
raise
failed.append(form.form_id)
error_messages[str(e)] += 1
else:
succeeded.append(form.form_id)
if not options["dryrun"]:
print("%s / %s forms successfully processed, %s failures" %
(len(succeeded), len(succeeded) + len(failed), len(failed)))
if error_messages:
print("The following errors were seen: \n%s" %
("\n".join("%s: %s" % (v, k)
for k, v in error_messages.items())))
def reprocess_archive_stubs():
# Check for archive stubs
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from couchforms.models import UnfinishedArchiveStub
stubs = UnfinishedArchiveStub.objects.filter(attempts__lt=3)
metrics_gauge('commcare.unfinished_archive_stubs', len(stubs))
start = time.time()
cutoff = start + timedelta(minutes=4).total_seconds()
for stub in stubs:
# Exit this task after 4 minutes so that tasks remain short
if time.time() > cutoff:
return
try:
xform = FormAccessors(stub.domain).get_form(form_id=stub.xform_id)
# If the history wasn't updated the first time around, run the whole thing again.
if not stub.history_updated:
FormAccessors.do_archive(xform, stub.archive, stub.user_id, trigger_signals=True)
# If the history was updated the first time around, just send the update to kafka
else:
FormAccessors.publish_archive_action_to_kafka(xform, stub.user_id, stub.archive)
except Exception:
# Errors should not prevent processing other stubs
notify_exception(None, "Error processing UnfinishedArchiveStub")
def bulk_undo(self, progress_bar=False):
chunk_size = 100
result = {
'processed': 0,
'skipped': 0,
'archived': 0,
}
form_ids = list(self.get_submission_queryset().values_list('form_id',
flat=True))
form_id_chunks = chunked(form_ids, chunk_size)
if progress_bar:
length = len(form_ids) / chunk_size
if len(form_ids) % chunk_size > 0:
length += 1
form_id_chunks = with_progress_bar(form_id_chunks, length=length)
for form_id_chunk in form_id_chunks:
archived_form_ids = []
for form in FormAccessors(self.domain).iter_forms(form_id_chunk):
result['processed'] += 1
if not form.is_normal or any(
[u.creates_case() for u in get_case_updates(form)]):
result['skipped'] += 1
continue
if not form.is_archived:
form.archive(user_id=SYSTEM_USER_ID)
result['archived'] += 1
archived_form_ids.append(form.form_id)
CaseRuleSubmission.objects.filter(
form_id__in=archived_form_ids).update(archived=True)
return result
def handle_unexpected_error(interface, instance, exception, message=None):
instance = _transform_instance_to_error(interface, exception, instance)
_notify_submission_error(instance, exception, instance.problem)
FormAccessors(interface.domain).save_new_form(instance)
def payload_doc(self, repeat_record):
return FormAccessors(repeat_record.domain).get_form(
repeat_record.payload_id)
def undelete_system_forms(domain, deleted_forms, deleted_cases):
"""The reverse of tag_system_forms_as_deleted; called on user.unretire()"""
to_undelete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=False)
FormAccessors(domain).soft_undelete_forms(to_undelete)
def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date):
to_delete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=True)
FormAccessors(domain).soft_delete_forms(to_delete, deletion_date, deletion_id)
def _is_safe_to_modify(form):
if form.domain != domain:
return False
case_ids = get_case_ids_from_form(form)
# all cases touched by the form and not already modified
for case in CaseAccessors(domain).iter_cases(case_ids - modified_cases):
if case.is_deleted != is_deletion:
# we can't delete/undelete this form - this would change the state of `case`
return False
# all cases touched by this form are deleted
return True
if is_deletion or Domain.get_by_name(domain).use_sql_backend:
all_forms = FormAccessors(domain).iter_forms(form_ids_to_modify)
else:
# accessor.iter_forms doesn't include deleted forms on the couch backend
all_forms = list(map(FormAccessors(domain).get_form, form_ids_to_modify))
return [form.form_id for form in all_forms if _is_safe_to_modify(form)]
@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date):
to_delete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=True)
FormAccessors(domain).soft_delete_forms(to_delete, deletion_date, deletion_id)
@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def undelete_system_forms(domain, deleted_forms, deleted_cases):
"""The reverse of tag_system_forms_as_deleted; called on user.unretire()"""
def _handle_duplicate(new_doc):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
and *must* include inline attachments
:returns: A two-tuple: `(<new form>, <duplicate form or None>)`
The new form may have a different `form_id` than `new_doc.form_id`.
"""
interface = FormProcessorInterface(new_doc.domain)
conflict_id = new_doc.form_id
try:
existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id)
except ResourceNotFound:
# Original form processing failed but left behind a form doc with no
# attachments. It's safe to delete this now since we're going to re-process
# the form anyway.
from couchforms.models import XFormInstance
XFormInstance.get_db().delete_doc(conflict_id)
return new_doc, None
try:
existing_md5 = existing_doc.xml_md5()
except MissingFormXml:
existing_md5 = None
if not existing_doc.is_error:
existing_doc.problem = 'Missing form.xml'
new_md5 = new_doc.xml_md5()
if existing_md5 != new_md5:
_soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False)
if new_doc.xmlns != existing_doc.xmlns:
# if the XMLNS has changed this probably isn't a form edit
# it could be a UUID clash (yes we've had that before)
# Assign a new ID to the form and process as normal + notify_admins
xform = interface.assign_new_id(new_doc)
_soft_assert(
False, "Potential UUID clash", {
'incoming_form_id': conflict_id,
'existing_form_id': existing_doc.form_id,
'new_form_id': xform.form_id,
'incoming_xmlns': new_doc.xmlns,
'existing_xmlns': existing_doc.xmlns,
'domain': new_doc.domain,
}
)
return xform, None
else:
if existing_doc.is_error and not existing_doc.initial_processing_complete:
# edge case from ICDS where a form errors and then future re-submissions of the same
# form do not have the same MD5 hash due to a bug on mobile:
# see https://dimagi-dev.atlassian.net/browse/ICDS-376
# since we have a new form and the old one was not successfully processed
# we can effectively ignore this form and process the new one as normal
if not interface.use_sql_domain:
new_doc._rev, existing_doc._rev = existing_doc._rev, new_doc._rev
interface.assign_new_id(existing_doc)
existing_doc.save()
return new_doc, None
else:
# if the form contents are not the same:
# - "Deprecate" the old form by making a new document with the same contents
# but a different ID and a doc_type of XFormDeprecated
# - Save the new instance to the previous document to preserve the ID
NotAllowed.check(new_doc.domain)
existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface)
return new_doc, existing_doc
else:
# follow standard dupe handling, which simply saves a copy of the form
# but a new doc_id, and a doc_type of XFormDuplicate
duplicate = interface.deduplicate_xform(new_doc)
return duplicate, existing_doc
def setUp(self):
super(TestFormArchiving, self).setUp()
self.casedb = CaseAccessors('test-domain')
self.formdb = FormAccessors('test-domain')
