def _handle_duplicate(existing_doc, instance, attachments, process): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments """ conflict_id = existing_doc.get_id existing_md5 = existing_doc.xml_md5() new_md5 = hashlib.md5(instance).hexdigest() if existing_md5 != new_md5: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID old_id = existing_doc._id new_id = XFormInstance.get_db().server.next_uuid() multi_lock_manager = process_xform(instance, attachments=attachments, process=process, _id=new_id) [(xform, _)] = multi_lock_manager # swap the two documents so the original ID now refers to the new one # and mark original as deprecated xform._id, existing_doc._id = old_id, new_id xform._rev, existing_doc._rev = existing_doc._rev, xform._rev # flag the old doc with metadata pointing to the new one existing_doc.doc_type = deprecation_type() existing_doc.orig_id = old_id # and give the new doc server data of the old one and some metadata xform.received_on = existing_doc.received_on xform.deprecated_form_id = existing_doc._id xform.edited_on = datetime.datetime.utcnow() multi_lock_manager.append( LockManager(existing_doc, acquire_lock_for_xform(old_id)) ) return multi_lock_manager else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate new_doc_id = uid.new() new_form, lock = create_xform(instance, attachments=attachments, _id=new_doc_id, process=process) new_form.doc_type = XFormDuplicate.__name__ dupe = XFormDuplicate.wrap(new_form.to_json()) dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id return MultiLockManager([LockManager(dupe, lock)])
def get_locked_forms(self): if self.existing_duplicate: # Lock docs with their original ID's (before they got switched during deprecation) new_id = self.existing_duplicate.form_id old_id = self.existing_duplicate.orig_id return MultiLockManager([ LockManager(self.submitted_form, self._get_form_lock(new_id)), LockManager(self.existing_duplicate, self._get_form_lock(old_id)), ]) else: return MultiLockManager([ LockManager(self.submitted_form, self._get_form_lock(self.submitted_form.form_id)) ])
def get_locked_forms(self): if self.existing_duplicate: # Lock docs with their original ID's (before they got switched during deprecation) old_id = self.existing_duplicate.form_id new_id = self.submitted_form.form_id assert old_id != new_id, 'Expecting form IDs to be different' return MultiLockManager([ LockManager(self.submitted_form, self._get_form_lock(new_id)), LockManager(self.existing_duplicate, self._get_form_lock(old_id)), ]) else: return MultiLockManager([ LockManager(self.submitted_form, self._get_form_lock(self.submitted_form.form_id)) ])
def change_trigger(self, changes_dict): doc_dict, lock = lock_manager( super(CasePillow, self).change_trigger(changes_dict)) if doc_dict['doc_type'] == 'CommCareCase-Deleted': if self.doc_exists(doc_dict): self.get_es().delete(path=self.get_doc_path_typed(doc_dict)) return None else: return LockManager(doc_dict, lock)
def create_xform(xml_string, attachments=None, _id=None, process=None): """ create but do not save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError """ from corehq.util.couch_helpers import CouchAttachmentsBuilder assert attachments is not None json_form = convert_xform_to_json(xml_string) adjust_datetimes(json_form) _id = (_id or _extract_meta_instance_id(json_form) or XFormInstance.get_db().server.next_uuid()) assert _id attachments_builder = CouchAttachmentsBuilder() attachments_builder.add( content=xml_string, name='form.xml', content_type='text/xml', ) for key, value in attachments.items(): attachments_builder.add( content=value, name=key, content_type=value.content_type, ) xform = XFormInstance( # form has to be wrapped {'form': json_form}, # other properties can be set post-wrap _id=_id, xmlns=json_form.get('@xmlns'), _attachments=attachments_builder.to_json(), received_on=datetime.datetime.utcnow(), ) # this had better not fail, don't think it ever has # if it does, nothing's saved and we get a 500 if process: process(xform) lock = acquire_lock_for_xform(_id) with ReleaseOnError(lock): if _id in XFormInstance.get_db(): raise DuplicateError() return LockManager(xform, lock)
def create_xform_from_xml(xml_string, _id=None, process=None): """ create and save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError - do not save form """ json_form = convert_xform_to_json(xml_string) _id = _id or _extract_meta_instance_id(json_form) kwargs = dict( _attachments=resource.encode_attachments({ "form.xml": { "content_type": "text/xml", "data": xml_string, }, }), form=json_form, xmlns=json_form.get('@xmlns'), received_on=datetime.datetime.utcnow(), ) if _id: kwargs['_id'] = _id xform = XFormInstance(**kwargs) try: if process: process(xform) except Exception: # if there's any problem with process just save what we had before # rather than whatever intermediate state `process` left it in xform = XFormInstance(**kwargs) raise finally: lock = acquire_lock_for_xform(_id) if _id else None with ReleaseOnError(lock): try: xform.save(encode_attachments=False) except ResourceConflict: raise DuplicateError() if not lock: lock = acquire_lock_for_xform(_id) return LockManager(xform.get_id, lock)
def create_and_lock_xform(instance, attachments=None, process=None, domain=None, _id=None): """ Save a new xform to couchdb in a thread-safe manner Returns a LockManager containing the new XFormInstance and its lock, or raises an exception if anything goes wrong. attachments is a dictionary of the request.FILES that are not the xform; key is parameter name, value is django MemoryFile object stream """ attachments = attachments or {} try: doc_id, lock = create_xform_from_xml(instance, process=process, _id=_id) except couchforms.XMLSyntaxError as e: doc = _log_hard_failure(instance, attachments, e) raise SubmissionError(doc) except DuplicateError: return _handle_id_conflict(instance, attachments, process=process, domain=domain) try: xform = XFormInstance.get(doc_id) for key, value in attachments.items(): xform.put_attachment(value, name=key, content_type=value.content_type, content_length=value.size) except Exception as e: logging.exception("Problem with form %s" % doc_id) # "rollback" by changing the doc_type to XFormError xform = XFormError.get(doc_id) xform.problem = unicode(e) xform.save() release_lock(lock, degrade_gracefully=True) lock = None return LockManager(xform, lock)
def _handle_duplicate(existing_doc, instance, attachments, process): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain """ conflict_id = existing_doc.get_id # compare md5s existing_md5 = existing_doc.xml_md5() new_md5 = hashlib.md5(instance).hexdigest() # if not same: # Deprecate old form (including changing ID) # to deprecate, copy new instance into a XFormDeprecated if existing_md5 != new_md5: doc_copy = XFormInstance.get_db().copy_doc(conflict_id) # get the doc back to avoid any potential bigcouch race conditions. # r=3 implied by class xfd = XFormDeprecated.get(doc_copy['id']) xfd.orig_id = conflict_id xfd.doc_type = XFormDeprecated.__name__ xfd.save() # after that delete the original document and resubmit. XFormInstance.get_db().delete_doc(conflict_id) return create_and_lock_xform(instance, attachments=attachments, process=process) else: # follow standard dupe handling new_doc_id = uid.new() new_form_id, lock = create_xform_from_xml(instance, _id=new_doc_id, process=process) # create duplicate doc # get and save the duplicate to ensure the doc types are set correctly # so that it doesn't show up in our reports dupe = XFormDuplicate.get(new_form_id) dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id dupe.save() return LockManager(dupe, lock)
def create_and_save_xform(xml_string): assert getattr(settings, 'UNIT_TESTING', False) xform, lock = create_xform(xml_string, attachments={}) with ReleaseOnError(lock): xform.save() return LockManager(xform.get_id, lock)
def reprocess_form(form, save=True, lock_form=True): interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform( form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache(domain=form.domain, lock=True, deleted_ok=True, xforms=[form]) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases( [form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format( type(e).__name__, six.text_type(e)) form = interface.xformerror_from_xform_instance( form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild( form, cases) ledgers = stock_result.models_to_save ledgers_updated = { ledger.ledger_reference for ledger in ledgers if ledger.is_saved() } if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL._publish_changes( ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state( **ledger.ledger_reference._asdict()) else:
def _get_lock(self, change): lock = self.document_class.get_obj_lock_by_id(change.id) lock.acquire() return LockManager(None, lock)
def _lock_manager(obj): if isinstance(obj, LockManager): return obj else: return LockManager(obj, None)
def get_locked_forms(self): return MultiLockManager([LockManager(self.submitted_form, self.lock)])
try: xform = XFormInstance.get(doc_id) for key, value in attachments.items(): xform.put_attachment(value, name=key, content_type=value.content_type, content_length=value.size) except Exception, e: logging.exception("Problem with form %s" % xform.get_id) # "rollback" by changing the doc_type to XFormError xform = XFormError.get(xform.get_id) xform.problem = unicode(e) xform.save() release_lock(lock, degrade_gracefully=True) lock = None return LockManager(xform, lock) def _has_errors(response, errors): return errors or "error" in response def _extract_id_from_raw_xml(xml): # the code this is replacing didn't deal with the error either # presumably because it's already been run once by the time it gets here _, json_form = xml2json.xml2json(xml) return _extract_meta_instance_id(json_form) or '' def _handle_id_conflict(instance, attachments, process, domain): """