def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
and *must* include inline attachments
"""
conflict_id = existing_doc.get_id
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
if existing_md5 != new_md5:
# if the form contents are not the same:
# - "Deprecate" the old form by making a new document with the same contents
# but a different ID and a doc_type of XFormDeprecated
# - Save the new instance to the previous document to preserve the ID
old_id = existing_doc._id
new_id = XFormInstance.get_db().server.next_uuid()
multi_lock_manager = process_xform(instance, attachments=attachments,
process=process, _id=new_id)
[(xform, _)] = multi_lock_manager
# swap the two documents so the original ID now refers to the new one
# and mark original as deprecated
xform._id, existing_doc._id = old_id, new_id
xform._rev, existing_doc._rev = existing_doc._rev, xform._rev
# flag the old doc with metadata pointing to the new one
existing_doc.doc_type = deprecation_type()
existing_doc.orig_id = old_id
# and give the new doc server data of the old one and some metadata
xform.received_on = existing_doc.received_on
xform.deprecated_form_id = existing_doc._id
xform.edited_on = datetime.datetime.utcnow()
multi_lock_manager.append(
LockManager(existing_doc,
acquire_lock_for_xform(old_id))
)
return multi_lock_manager
else:
# follow standard dupe handling, which simply saves a copy of the form
# but a new doc_id, and a doc_type of XFormDuplicate
new_doc_id = uid.new()
new_form, lock = create_xform(instance, attachments=attachments,
_id=new_doc_id, process=process)
new_form.doc_type = XFormDuplicate.__name__
dupe = XFormDuplicate.wrap(new_form.to_json())
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
return MultiLockManager([LockManager(dupe, lock)])
def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
and *must* include inline attachments
"""
conflict_id = existing_doc.get_id
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
if existing_md5 != new_md5:
# if the form contents are not the same:
# - "Deprecate" the old form by making a new document with the same contents
# but a different ID and a doc_type of XFormDeprecated
# - Save the new instance to the previous document to preserve the ID
old_id = existing_doc._id
new_id = XFormInstance.get_db().server.next_uuid()
multi_lock_manager = process_xform(instance, attachments=attachments,
process=process, _id=new_id)
[(xform, _)] = multi_lock_manager
# swap the two documents so the original ID now refers to the new one
# and mark original as deprecated
xform._id, existing_doc._id = old_id, new_id
xform._rev, existing_doc._rev = existing_doc._rev, xform._rev
# flag the old doc with metadata pointing to the new one
existing_doc.doc_type = deprecation_type()
existing_doc.orig_id = old_id
# and give the new doc server data of the old one and some metadata
xform.received_on = existing_doc.received_on
xform.deprecated_form_id = existing_doc._id
xform.edited_on = datetime.datetime.utcnow()
multi_lock_manager.append(
LockManager(existing_doc,
acquire_lock_for_xform(old_id))
)
return multi_lock_manager
else:
# follow standard dupe handling, which simply saves a copy of the form
# but a new doc_id, and a doc_type of XFormDuplicate
new_doc_id = uid.new()
new_form, lock = create_xform(instance, attachments=attachments,
_id=new_doc_id, process=process)
new_form.doc_type = XFormDuplicate.__name__
dupe = XFormDuplicate.wrap(new_form.to_json())
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
return MultiLockManager([LockManager(dupe, lock)])
def copy(cls, instance):
"""
Create a backup copy of a instance object, returning the id of the
newly created document
"""
instance_json = instance.to_json()
backup_id = uid.new()
instance_json["_id"] = backup_id
instance_json.pop("_rev")
get_db().save_doc(instance_json)
return backup_id
def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
and *must* include inline attachments
"""
conflict_id = existing_doc.get_id
# compare md5s
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
# if not same:
# Deprecate old form (including changing ID)
# to deprecate, copy new instance into a XFormDeprecated
# todo: save transactionally like the rest of the code
if existing_md5 != new_md5:
old_id = existing_doc.get_id
new_id = XFormInstance.get_db().server.next_uuid()
multi_lock_manager = process_xform(instance, attachments=attachments,
process=process, _id=new_id)
[(xform, _)] = multi_lock_manager
# swap the two documents so the original ID now refers to the new one
# and mark original as deprecated
xform._id, existing_doc._id = old_id, new_id
xform._rev, existing_doc._rev = existing_doc._rev, xform._rev
existing_doc.doc_type = XFormDeprecated.__name__
existing_doc.orig_id = old_id
multi_lock_manager.append(
LockManager(existing_doc,
acquire_lock_for_xform(old_id))
)
return multi_lock_manager
else:
# follow standard dupe handling
new_doc_id = uid.new()
new_form, lock = create_xform(instance, attachments=attachments,
_id=new_doc_id, process=process)
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
new_form.doc_type = XFormDuplicate.__name__
dupe = XFormDuplicate.wrap(new_form.to_json())
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
return MultiLockManager([LockManager(dupe, lock)])
def _handle_id_conflict(instance, attachments, process, domain):
"""
For id conflicts, we check if the files contain exactly the same content,
If they do, we just log this as a dupe. If they don't, we deprecate the
previous form and overwrite it with the new form's contents.
"""
conflict_id = _extract_id_from_raw_xml(instance)
# get old document
existing_doc = XFormInstance.get_db().get(conflict_id, attachments=True)
assert domain
if existing_doc.get('domain') != domain\
or existing_doc.get('doc_type') not in doc_types():
return process_xform(instance, attachments=attachments,
process=process, _id=uid.new())
else:
existing_doc = XFormInstance.wrap(existing_doc)
return _handle_duplicate(existing_doc, instance, attachments, process)
def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
"""
conflict_id = existing_doc.get_id
# compare md5s
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
# if not same:
# Deprecate old form (including changing ID)
# to deprecate, copy new instance into a XFormDeprecated
if existing_md5 != new_md5:
doc_copy = XFormInstance.get_db().copy_doc(conflict_id)
# get the doc back to avoid any potential bigcouch race conditions.
# r=3 implied by class
xfd = XFormDeprecated.get(doc_copy['id'])
xfd.orig_id = conflict_id
xfd.doc_type = XFormDeprecated.__name__
xfd.save()
# after that delete the original document and resubmit.
XFormInstance.get_db().delete_doc(conflict_id)
return create_and_lock_xform(instance,
attachments=attachments,
process=process)
else:
# follow standard dupe handling
new_doc_id = uid.new()
new_form_id, lock = create_xform_from_xml(instance,
_id=new_doc_id,
process=process)
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
dupe = XFormDuplicate.get(new_form_id)
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
dupe.save()
return LockManager(dupe, lock)
def testRandomFunction(self):
# since this is a random statistical test there is a chance
# it could readily fail. The threshold was chosen to be
# what seems to be well out of normal range of firing, but
# in randomness anything is possible.
test_size = 100000
threshold = 0.003
probabilities = [0, 0.02, 0.5, 0.9, 1]
for probability in probabilities:
truecount = 0
falsecount = 0
while truecount + falsecount < test_size:
if predictable_random(uid.new(), probability):
truecount += 1
else:
falsecount += 1
calculated_probability = float(truecount) / float(test_size)
self.assertTrue(probability - threshold < calculated_probability)
self.assertTrue(probability + threshold > calculated_probability)
def _handle_duplicate(existing_doc, instance, attachments, process):
"""
Handle duplicate xforms and xform editing ('deprecation')
existing doc *must* be validated as an XFormInstance in the right domain
"""
conflict_id = existing_doc.get_id
# compare md5s
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
# if not same:
# Deprecate old form (including changing ID)
# to deprecate, copy new instance into a XFormDeprecated
if existing_md5 != new_md5:
doc_copy = XFormInstance.get_db().copy_doc(conflict_id)
# get the doc back to avoid any potential bigcouch race conditions.
# r=3 implied by class
xfd = XFormDeprecated.get(doc_copy['id'])
xfd.orig_id = conflict_id
xfd.doc_type = XFormDeprecated.__name__
xfd.save()
# after that delete the original document and resubmit.
XFormInstance.get_db().delete_doc(conflict_id)
return create_and_lock_xform(instance, attachments=attachments,
process=process)
else:
# follow standard dupe handling
new_doc_id = uid.new()
new_form_id, lock = create_xform_from_xml(instance, _id=new_doc_id,
process=process)
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
dupe = XFormDuplicate.get(new_form_id)
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
dupe.save()
return LockManager(dupe, lock)
def _handle_id_conflict(instance, attachments, process, domain):
"""
For id conflicts, we check if the files contain exactly the same content,
If they do, we just log this as a dupe. If they don't, we deprecate the
previous form and overwrite it with the new form's contents.
"""
assert domain
conflict_id = _extract_id_from_raw_xml(instance)
existing_doc = XFormInstance.get_db().get(conflict_id, attachments=True)
if existing_doc.get('domain') != domain or existing_doc.get('doc_type') not in doc_types():
# the same form was submitted to two domains, or a form was submitted with
# an ID that belonged to a different doc type. these are likely developers
# manually testing or broken API users. just resubmit with a generated ID.
return process_xform(instance, attachments=attachments,
process=process, _id=uid.new())
else:
# It looks like a duplicate/edit in the same domain so pursue that workflow.
existing_doc = XFormInstance.wrap(existing_doc)
return _handle_duplicate(existing_doc, instance, attachments, process)
def _handle_id_conflict(instance, attachments, process, domain):
"""
For id conflicts, we check if the files contain exactly the same content,
If they do, we just log this as a dupe. If they don't, we deprecate the
previous form and overwrite it with the new form's contents.
"""
assert domain
conflict_id = _extract_id_from_raw_xml(instance)
existing_doc = XFormInstance.get_db().get(conflict_id, attachments=True)
if existing_doc.get('domain') != domain or existing_doc.get('doc_type') not in doc_types():
# the same form was submitted to two domains, or a form was submitted with
# an ID that belonged to a different doc type. these are likely developers
# manually testing or broken API users. just resubmit with a generated ID.
return process_xform(instance, attachments=attachments,
process=process, _id=uid.new())
else:
# It looks like a duplicate/edit in the same domain so pursue that workflow.
existing_doc = XFormInstance.wrap(existing_doc)
return _handle_duplicate(existing_doc, instance, attachments, process)
def from_xform(cls, doc):
"""
Create an encounter object from an xform document.
"""
type = ENCOUNTERS_BY_XMLNS[doc.namespace].type if doc.namespace in ENCOUNTERS_BY_XMLNS else doc.namespace
visit_date = Encounter.get_visit_date(doc)
metadata = {}
if doc.metadata:
metadata = doc.metadata.to_dict()
return Encounter(
_id=uid.new(),
created=datetime.utcnow(),
edited=datetime.utcnow(),
visit_date=visit_date,
type=type,
is_deprecated=False,
_metadata=metadata,
xform_id=doc["_id"],
)
def _handle_id_conflict(instance, attachments, process, domain):
"""
For id conflicts, we check if the files contain exactly the same content,
If they do, we just log this as a dupe. If they don't, we deprecate the
previous form and overwrite it with the new form's contents.
"""
conflict_id = _extract_id_from_raw_xml(instance)
# get old document
existing_doc = XFormInstance.get_db().get(conflict_id)
assert domain
if existing_doc.get('domain') != domain\
or existing_doc.get('doc_type') not in doc_types():
# exit early
return create_and_lock_xform(instance,
attachments=attachments,
process=process,
_id=uid.new())
else:
existing_doc = XFormInstance.wrap(existing_doc)
return _handle_duplicate(existing_doc, instance, attachments, process)
def testCustomResponse(self):
template_file = os.path.join(os.path.dirname(__file__), "data", "referral.xml")
with open(template_file) as f:
template = f.read()
c = Client()
now = datetime.now().date()
def _post_and_confirm(form, count):
response = c.post("/phone/post/", data=form, content_type="text/xml")
self.assertEqual(200, response.status_code)
self.assertTrue(("<FormsSubmittedToday>%s</FormsSubmittedToday>" % count) in response.content)
self.assertTrue(("<TotalFormsSubmitted>%s</TotalFormsSubmitted>" % count) in response.content)
for i in range(5):
form = template % {"date": date_to_xml_string(now), "uid": uid.new()}
_post_and_confirm(form, i + 1)
for i in range(5):
# make sure past submissions also show up as today
form = template % {"date": date_to_xml_string(now - timedelta(days=i)), "uid": uid.new()}
_post_and_confirm(form, i + 6)
# if not same:
# Deprecate old form (including changing ID)
# to deprecate, copy new instance into a XFormDeprecated
if existing_md5 != new_md5:
doc_copy = XFormInstance.get_db().copy_doc(conflict_id)
xfd = XFormDeprecated.get(doc_copy['id'])
xfd.orig_id = conflict_id
xfd.doc_type = XFormDeprecated.__name__
xfd.save()
# after that delete the original document and resubmit.
XFormInstance.get_db().delete_doc(conflict_id)
return post_xform_to_couch(instance, attachments=attachments)
else:
# follow standard dupe handling
new_doc_id = uid.new()
response, errors = post_from_settings(instance,
{"uid": new_doc_id})
if not _has_errors(response, errors):
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
dupe = XFormDuplicate.get(response)
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
dupe.save()
return dupe
else:
# how badly do we care about this?
raise CouchFormException(
"Problem POSTing form to couch! errors/response: %s/%s"
% (errors, response))
def __init__(self, patient, pregnancy):
self.patient = patient
self._pregnancy = pregnancy
self._id = uid.new()
raise
else:
raise XFormException("Problem POSTing form to couch! errors/response: %s/%s" % (errors, response))
except RequestFailed, e:
if e.status_int == 409:
# this is an update conflict, i.e. the uid in the form was the same.
# log it and flag it.
def _extract_id_from_raw_xml(xml):
# TODO: this is brittle as hell. Fix.
_PATTERNS = (r"<uid>(\w+)</uid>", r"<uuid>(\w+)</uuid>")
for pattern in _PATTERNS:
if re.search(pattern, xml): return re.search(pattern, xml).groups()[0]
logging.error("Unable to find conflicting matched uid in form: %s" % xml)
return ""
conflict_id = _extract_id_from_raw_xml(instance)
new_doc_id = uid.new()
response, errors = post_from_settings(instance, {"uid": new_doc_id})
if not _has_errors(response, errors):
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
dupe = CXFormDuplicate.get(response)
dupe.release_lock()
dupe.save()
return dupe
else:
# how badly do we care about this?
raise XFormException("Problem POSTing form to couch! errors/response: %s/%s" % (errors, response))
else:
raise
def _handle_id_conflict(instance, attachments):
"""
For id conflicts, we check if the files contain exactly the same content,
If they do, we just log this as a dupe. If they don't, we deprecate the
previous form and overwrite it with the new form's contents.
"""
def _extract_id_from_raw_xml(xml):
# this is the standard openrosa way of doing things
parsed = etree.XML(xml)
meta_ns = "http://openrosa.org/jr/xforms"
val = parsed.find("{%(ns)s}meta/{%(ns)s}instanceID" % \
{"ns": meta_ns})
if val is not None and val.text:
return val.text
# if we get here search more creatively for some of the older
# formats
_PATTERNS = (r"<instanceID>([\w-]+)</instanceID>",
r"<uid>([\w-]+)</uid>",
r"<uuid>([\w-]+)</uuid>")
for pattern in _PATTERNS:
if re.search(pattern, xml):
return re.search(pattern, xml).groups()[0]
logging.error("Unable to find conflicting matched uid in form: %s" % xml)
return ""
conflict_id = _extract_id_from_raw_xml(instance)
# get old document
existing_doc = XFormInstance.get(conflict_id)
# compare md5s
existing_md5 = existing_doc.xml_md5()
new_md5 = hashlib.md5(instance).hexdigest()
# if not same:
# Deprecate old form (including changing ID)
# to deprecate, copy new instance into a XFormDeprecated
if existing_md5 != new_md5:
doc_copy = XFormInstance.get_db().copy_doc(conflict_id)
# get the doc back to avoid any potential bigcouch race conditions.
# r=3 implied by class
xfd = XFormDeprecated.get(doc_copy['id'])
xfd.orig_id = conflict_id
xfd.doc_type=XFormDeprecated.__name__
xfd.save()
# after that delete the original document and resubmit.
XFormInstance.get_db().delete_doc(conflict_id)
return post_xform_to_couch(instance, attachments=attachments)
else:
# follow standard dupe handling
new_doc_id = uid.new()
response, errors = post_from_settings(instance, {"uid": new_doc_id})
if not _has_errors(response, errors):
# create duplicate doc
# get and save the duplicate to ensure the doc types are set correctly
# so that it doesn't show up in our reports
dupe = XFormDuplicate.get(response)
dupe.problem = "Form is a duplicate of another! (%s)" % conflict_id
dupe.save()
return dupe
else:
# how badly do we care about this?
raise CouchFormException("Problem POSTing form to couch! errors/response: %s/%s" % (errors, response))