def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
and *must* include inline attachments
"""
conflict_id = existing_doc.get_id
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
if existing_md5 != new_md5:
# if the form contents are not the same:
# - "Deprecate" the old form by making a new document with the same contents
# but a different ID and a doc_type of XFormDeprecated
# - Save the new instance to the previous document to preserve the ID
old_id = existing_doc._id
new_id = XFormInstance.get_db().server.next_uuid()
multi_lock_manager = process_xform(instance, attachments=attachments,
process=process, _id=new_id)
[(xform, _)] = multi_lock_manager
# swap the two documents so the original ID now refers to the new one
# and mark original as deprecated
xform._id, existing_doc._id = old_id, new_id
xform._rev, existing_doc._rev = existing_doc._rev, xform._rev
# flag the old doc with metadata pointing to the new one
existing_doc.doc_type = deprecation_type()
existing_doc.orig_id = old_id
# and give the new doc server data of the old one and some metadata
xform.received_on = existing_doc.received_on
xform.deprecated_form_id = existing_doc._id
xform.edited_on = datetime.datetime.utcnow()
multi_lock_manager.append(
LockManager(existing_doc,
acquire_lock_for_xform(old_id))
)
return multi_lock_manager
else:
# follow standard dupe handling, which simply saves a copy of the form
# but a new doc_id, and a doc_type of XFormDuplicate
new_doc_id = uid.new()
new_form, lock = create_xform(instance, attachments=attachments,
_id=new_doc_id, process=process)
new_form.doc_type = XFormDuplicate.__name__
dupe = XFormDuplicate.wrap(new_form.to_json())
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
return MultiLockManager([LockManager(dupe, lock)])
def get_locked_forms(self):
if self.existing_duplicate:
# Lock docs with their original ID's (before they got switched during deprecation)
new_id = self.existing_duplicate.form_id
old_id = self.existing_duplicate.orig_id
return MultiLockManager([
LockManager(self.submitted_form, self._get_form_lock(new_id)),
LockManager(self.existing_duplicate,
self._get_form_lock(old_id)),
])
else:
return MultiLockManager([
LockManager(self.submitted_form,
self._get_form_lock(self.submitted_form.form_id))
])
def get_locked_forms(self):
if self.existing_duplicate:
# Lock docs with their original ID's (before they got switched during deprecation)
old_id = self.existing_duplicate.form_id
new_id = self.submitted_form.form_id
assert old_id != new_id, 'Expecting form IDs to be different'
return MultiLockManager([
LockManager(self.submitted_form, self._get_form_lock(new_id)),
LockManager(self.existing_duplicate,
self._get_form_lock(old_id)),
])
else:
return MultiLockManager([
LockManager(self.submitted_form,
self._get_form_lock(self.submitted_form.form_id))
])
def change_trigger(self, changes_dict):
doc_dict, lock = lock_manager(
super(CasePillow, self).change_trigger(changes_dict))
if doc_dict['doc_type'] == 'CommCareCase-Deleted':
if self.doc_exists(doc_dict):
self.get_es().delete(path=self.get_doc_path_typed(doc_dict))
return None
else:
return LockManager(doc_dict, lock)
def create_xform(xml_string, attachments=None, _id=None, process=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
"""
from corehq.util.couch_helpers import CouchAttachmentsBuilder
assert attachments is not None
json_form = convert_xform_to_json(xml_string)
adjust_datetimes(json_form)
_id = (_id or _extract_meta_instance_id(json_form)
or XFormInstance.get_db().server.next_uuid())
assert _id
attachments_builder = CouchAttachmentsBuilder()
attachments_builder.add(
content=xml_string,
name='form.xml',
content_type='text/xml',
)
for key, value in attachments.items():
attachments_builder.add(
content=value,
name=key,
content_type=value.content_type,
)
xform = XFormInstance(
# form has to be wrapped
{'form': json_form},
# other properties can be set post-wrap
_id=_id,
xmlns=json_form.get('@xmlns'),
_attachments=attachments_builder.to_json(),
received_on=datetime.datetime.utcnow(),
)
# this had better not fail, don't think it ever has
# if it does, nothing's saved and we get a 500
if process:
process(xform)
lock = acquire_lock_for_xform(_id)
with ReleaseOnError(lock):
if _id in XFormInstance.get_db():
raise DuplicateError()
return LockManager(xform, lock)
def create_xform_from_xml(xml_string, _id=None, process=None):
"""
create and save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
- do not save form
"""
json_form = convert_xform_to_json(xml_string)
_id = _id or _extract_meta_instance_id(json_form)
kwargs = dict(
_attachments=resource.encode_attachments({
"form.xml": {
"content_type": "text/xml",
"data": xml_string,
},
}),
form=json_form,
xmlns=json_form.get('@xmlns'),
received_on=datetime.datetime.utcnow(),
)
if _id:
kwargs['_id'] = _id
xform = XFormInstance(**kwargs)
try:
if process:
process(xform)
except Exception:
# if there's any problem with process just save what we had before
# rather than whatever intermediate state `process` left it in
xform = XFormInstance(**kwargs)
raise
finally:
lock = acquire_lock_for_xform(_id) if _id else None
with ReleaseOnError(lock):
try:
xform.save(encode_attachments=False)
except ResourceConflict:
raise DuplicateError()
if not lock:
lock = acquire_lock_for_xform(_id)
return LockManager(xform.get_id, lock)
def create_and_lock_xform(instance,
attachments=None,
process=None,
domain=None,
_id=None):
"""
Save a new xform to couchdb in a thread-safe manner
Returns a LockManager containing the new XFormInstance and its lock,
or raises an exception if anything goes wrong.
attachments is a dictionary of the request.FILES that are not the xform;
key is parameter name, value is django MemoryFile object stream
"""
attachments = attachments or {}
try:
doc_id, lock = create_xform_from_xml(instance,
process=process,
_id=_id)
except couchforms.XMLSyntaxError as e:
doc = _log_hard_failure(instance, attachments, e)
raise SubmissionError(doc)
except DuplicateError:
return _handle_id_conflict(instance,
attachments,
process=process,
domain=domain)
try:
xform = XFormInstance.get(doc_id)
for key, value in attachments.items():
xform.put_attachment(value,
name=key,
content_type=value.content_type,
content_length=value.size)
except Exception as e:
logging.exception("Problem with form %s" % doc_id)
# "rollback" by changing the doc_type to XFormError
xform = XFormError.get(doc_id)
xform.problem = unicode(e)
xform.save()
release_lock(lock, degrade_gracefully=True)
lock = None
return LockManager(xform, lock)
def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
"""
conflict_id = existing_doc.get_id
# compare md5s
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
# if not same:
# Deprecate old form (including changing ID)
# to deprecate, copy new instance into a XFormDeprecated
if existing_md5 != new_md5:
doc_copy = XFormInstance.get_db().copy_doc(conflict_id)
# get the doc back to avoid any potential bigcouch race conditions.
# r=3 implied by class
xfd = XFormDeprecated.get(doc_copy['id'])
xfd.orig_id = conflict_id
xfd.doc_type = XFormDeprecated.__name__
xfd.save()
# after that delete the original document and resubmit.
XFormInstance.get_db().delete_doc(conflict_id)
return create_and_lock_xform(instance,
attachments=attachments,
process=process)
else:
# follow standard dupe handling
new_doc_id = uid.new()
new_form_id, lock = create_xform_from_xml(instance,
_id=new_doc_id,
process=process)
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
dupe = XFormDuplicate.get(new_form_id)
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
dupe.save()
return LockManager(dupe, lock)
def create_and_save_xform(xml_string):
assert getattr(settings, 'UNIT_TESTING', False)
xform, lock = create_xform(xml_string, attachments={})
with ReleaseOnError(lock):
xform.save()
return LockManager(xform.get_id, lock)
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
def _get_lock(self, change):
lock = self.document_class.get_obj_lock_by_id(change.id)
lock.acquire()
return LockManager(None, lock)
def _lock_manager(obj):
if isinstance(obj, LockManager):
return obj
else:
return LockManager(obj, None)
def get_locked_forms(self):
return MultiLockManager([LockManager(self.submitted_form, self.lock)])
try:
xform = XFormInstance.get(doc_id)
for key, value in attachments.items():
xform.put_attachment(value,
name=key,
content_type=value.content_type,
content_length=value.size)
except Exception, e:
logging.exception("Problem with form %s" % xform.get_id)
# "rollback" by changing the doc_type to XFormError
xform = XFormError.get(xform.get_id)
xform.problem = unicode(e)
xform.save()
release_lock(lock, degrade_gracefully=True)
lock = None
return LockManager(xform, lock)
def _has_errors(response, errors):
return errors or "error" in response
def _extract_id_from_raw_xml(xml):
# the code this is replacing didn't deal with the error either
# presumably because it's already been run once by the time it gets here
_, json_form = xml2json.xml2json(xml)
return _extract_meta_instance_id(json_form) or ''
def _handle_id_conflict(instance, attachments, process, domain):
"""