def test_inject_instanceid(self):
"""
Test that 1 and only 1 instance id exists or is injected
"""
instance = Instance.objects.all().reverse()[0]
xml_str = self.__load_fixture("..", "fixtures", "tutorial",
"instances",
"tutorial_2012-06-27_11-27-53.xml")
# test that we dont have an instance id
uuid = get_uuid_from_xml(xml_str)
self.assertIsNone(uuid)
injected_xml_str = inject_instanceid(xml_str, instance.uuid)
# check that xml has the instanceid tag
uuid = get_uuid_from_xml(injected_xml_str)
self.assertEqual(uuid, instance.uuid)
def test_inject_instanceid(self):
"""
Test that 1 and only 1 instance id exists or is injected
"""
instance = Instance.objects.all().reverse()[0]
xml_str = self.__load_fixture("..", "fixtures", "tutorial",
"instances",
"tutorial_2012-06-27_11-27-53.xml")
# test that we dont have an instance id
uuid = get_uuid_from_xml(xml_str)
self.assertIsNone(uuid)
injected_xml_str = inject_instanceid(xml_str, instance.uuid)
# check that xml has the instanceid tag
uuid = get_uuid_from_xml(injected_xml_str)
self.assertEqual(uuid, instance.uuid)
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
existing_instance_count = Instance.objects.filter(
xml=xml, xform__user=xform.user).count()
if existing_instance_count > 0:
existing_instance = Instance.objects.filter(
xml=xml, xform__user=xform.user)[0]
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status, date_created_override)
# commit all changes
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
def inject_instanceid(xml_str, uuid):
if get_uuid_from_xml(xml_str) is None:
xml = clean_and_parse_xml(xml_str)
children = xml.childNodes
if children.length == 0:
raise ValueError(_("XML string must have a survey element."))
# check if we have a meta tag
survey_node = children.item(0)
meta_tags = [
n for n in survey_node.childNodes
if n.nodeType == Node.ELEMENT_NODE
and n.tagName.lower() == "meta"]
if len(meta_tags) == 0:
meta_tag = xml.createElement("meta")
xml.documentElement.appendChild(meta_tag)
else:
meta_tag = meta_tags[0]
# check if we have an instanceID tag
uuid_tags = [
n for n in meta_tag.childNodes
if n.nodeType == Node.ELEMENT_NODE
and n.tagName == "instanceID"]
if len(uuid_tags) == 0:
uuid_tag = xml.createElement("instanceID")
meta_tag.appendChild(uuid_tag)
else:
uuid_tag = uuid_tags[0]
# insert meta and instanceID
text_node = xml.createTextNode(u"uuid:%s" % uuid)
uuid_tag.appendChild(text_node)
return xml.toxml()
return xml_str
def inject_instanceid(xml_str, uuid):
if get_uuid_from_xml(xml_str) is None:
xml = clean_and_parse_xml(xml_str)
children = xml.childNodes
if children.length == 0:
raise ValueError(_("XML string must have a survey element."))
# check if we have a meta tag
survey_node = children.item(0)
meta_tags = [
n for n in survey_node.childNodes
if n.nodeType == Node.ELEMENT_NODE and n.tagName.lower() == "meta"
]
if len(meta_tags) == 0:
meta_tag = xml.createElement("meta")
xml.documentElement.appendChild(meta_tag)
else:
meta_tag = meta_tags[0]
# check if we have an instanceID tag
uuid_tags = [
n for n in meta_tag.childNodes
if n.nodeType == Node.ELEMENT_NODE and n.tagName == "instanceID"
]
if len(uuid_tags) == 0:
uuid_tag = xml.createElement("instanceID")
meta_tag.appendChild(uuid_tag)
else:
uuid_tag = uuid_tags[0]
# insert meta and instanceID
text_node = xml.createTextNode(u"uuid:%s" % uuid)
uuid_tag.appendChild(text_node)
return xml.toxml()
return xml_str
def _set_uuid(self):
# pylint: disable=no-member, attribute-defined-outside-init
if self.xml and not self.uuid:
# pylint: disable=no-member
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def _set_uuid(self):
# pylint: disable=no-member, attribute-defined-outside-init
if self.xml and not self.uuid:
# pylint: disable=no-member
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def _set_uuid(self):
# pylint: disable=E1101, E0203
if self.xml and not self.uuid:
# pylint: disable=E1101
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xml_hash = Instance.get_hash(xml)
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
# Dorey's rule from 2012 (commit 890a67aa):
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
if xform.has_start_time:
# XML matches are identified by identical content hash OR, when a
# content hash is not present, by string comparison of the full
# content, which is slow! Use the management command
# `populate_xml_hashes_for_instances` to hash existing submissions
existing_instance = Instance.objects.filter(
Q(xml_hash=xml_hash)
| Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
).first()
else:
existing_instance = None
# get new and deprecated uuid's
new_uuid = get_uuid_from_xml(xml)
if existing_instance:
# ensure we have saved the extra attachments
any_new_attachment = save_attachments(existing_instance, media_files)
if not any_new_attachment:
raise DuplicateInstance()
else:
# Update Mongo via the related ParsedInstance
existing_instance.parsed_instance.save(async=False)
return existing_instance
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status, date_created_override)
return instance
def test_get_uuid_from_xml(self):
with open(
os.path.join(os.path.dirname(__file__), "..", "fixtures",
"tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid.xml"),
"r") as xml_file:
xml_str = xml_file.read()
instanceID = get_uuid_from_xml(xml_str)
self.assertEqual(instanceID, "729f173c688e482486a48661700455ff")
def test_get_uuid_from_xml(self):
with open(
os.path.join(
os.path.dirname(__file__), "..", "fixtures", "tutorial",
"instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml"),
"r") as xml_file:
xml_str = xml_file.read()
instanceID = get_uuid_from_xml(xml_str)
self.assertEqual(instanceID, "729f173c688e482486a48661700455ff")
def create_instance(fsxfid, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None, site=None, fs_proj_xf=None, proj_id=None, xform=None, flagged_instance=None):
with transaction.atomic():
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
xml = xml_file.read()
xml_hash = Instance.get_hash(xml)
if xform.has_start_time:
# XML matches are identified by identical content hash OR, when a
# content hash is not present, by string comparison of the full
# content, which is slow! Use the management command
# `populate_xml_hashes_for_instances` to hash existing submissions
existing_instance = Instance.objects.filter(
Q(xml_hash=xml_hash) |
Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
).first()
else:
existing_instance = None
new_uuid = get_uuid_from_xml(xml)
uuid = new_uuid
if existing_instance:
# ensure we have saved the extra attachments
any_new_attachment = save_attachments(existing_instance, media_files)
if not any_new_attachment:
raise DuplicateInstance()
else:
# Update Mongo via the related ParsedInstance
if fs_proj_xf:
fs_poj_id = str(fs_proj_xf)
else:
fs_poj_id = ""
pi, created = FieldSightParsedInstance.get_or_create(existing_instance,
update_data={'fs_uuid': str(fsxfid), 'fs_status': 0,
'fs_site': site,
'fs_project': proj_id,
'fs_project_uuid': fs_poj_id})
print()
return existing_instance
else:
if fsxfid is None:
fsxfid = ""
if site is None:
site = ""
instance = save_submission(xform, xml, media_files, uuid,
submitted_by, status,
date_created_override, str(fsxfid), str(site), fs_proj_xf, proj_id, flagged_instance)
return instance
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xml_hash = Instance.get_hash(xml)
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
# Dorey's rule from 2012 (commit 890a67aa):
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
if xform.has_start_time:
# XML matches are identified by identical content hash OR, when a
# content hash is not present, by string comparison of the full
# content, which is slow! Use the management command
# `populate_xml_hashes_for_instances` to hash existing submissions
existing_instance = Instance.objects.filter(
Q(xml_hash=xml_hash) | Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
).first()
else:
existing_instance = None
# get new and deprecated uuid's
new_uuid = get_uuid_from_xml(xml)
if existing_instance:
# ensure we have saved the extra attachments
any_new_attachment = save_attachments(existing_instance, media_files)
if not any_new_attachment:
raise DuplicateInstance()
else:
# Update Mongo via the related ParsedInstance
existing_instance.parsed_instance.save(async=False)
return existing_instance
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override)
return instance
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
instance = None
user = get_object_or_404(User, username=username.lower())
submitted_by = request.user \
if request and request.user.is_authenticated() else user
if username:
username = username.lower()
xml = xml_file.read()
#print 'xml of form'+str(xml)
#valid = check_custom_form_validation(xml,request,username)
valid = True
if not valid:
#print 'Form is not Valid..... '
transaction.rollback()
raise ValueError('need to register your tubewell first...')
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform, user)
existing_instance_count = Instance.objects.filter(
xml=xml, xform__user=xform.user).count()
if existing_instance_count > 0:
existing_instance = Instance.objects.filter(
xml=xml, xform__user=xform.user)[0]
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f,
mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
if is_exist_farmer_paravet_ait(xml, request, username, user) == 1:
transaction.commit()
raise DuplicateFarmer()
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status, date_created_override)
# commit all changes
call_parave_ai_reg_api(xml, request, username, user)
send_push_noti_after_profile_update(xml)
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is no username and a uuid, submitting a touchform.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
if username:
username = username.lower()
xml = xml_file.read()
is_touchform = False
# check alternative form submission ids
if not uuid:
# parse UUID from uploaded XML
split_xml = uuid_regex.split(xml)
# check that xml has UUID, then it is a crowdform
if len(split_xml) > 1:
uuid = split_xml[1]
else:
# is a touchform
is_touchform = True
if not username and not uuid:
raise InstanceInvalidUserError()
if uuid:
# try find the form by its uuid which is the ideal condition
if XForm.objects.filter(uuid=uuid).count() > 0:
xform = XForm.objects.get(uuid=uuid)
xform_username = xform.user.username
if xform_username != username and not xform.is_crowd_form \
and not is_touchform:
raise IsNotCrowdformError()
username = xform_username
# Else, since we have a username, the Instance creation logic will
# handle checking for the forms existence by its id_string
if username and request and request.user.is_authenticated():
id_string = get_id_string_from_xml_str(xml)
xform = XForm.objects.get(
id_string=id_string, user__username=username)
if not xform.is_crowd_form and not is_touchform \
and xform.user.profile.require_auth \
and xform.user != request.user:
raise PermissionDenied(
_(u"%(request_user)s is not allowed to make submissions "
u"to %(form_user)s's %(form_title)s form." % {
'request_user': request.user,
'form_user': xform.user,
'form_title': xform.title}))
user = get_object_or_404(User, username=username)
existing_instance_count = Instance.objects.filter(
xml=xml, user=user).count()
if existing_instance_count == 0:
proceed_to_create_instance = True
else:
existing_instance = Instance.objects.filter(xml=xml, user=user)[0]
if existing_instance.xform and\
not existing_instance.xform.has_start_time:
proceed_to_create_instance = True
else:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
proceed_to_create_instance = False
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
if proceed_to_create_instance:
# check if its an edit submission
old_uuid = get_deprecated_uuid_from_xml(xml)
instances = Instance.objects.filter(uuid=old_uuid)
if not date_created_override:
date_created_override = get_submission_date_from_xml(xml)
if instances:
instance = instances[0]
InstanceHistory.objects.create(
xml=instance.xml, xform_instance=instance, uuid=old_uuid)
instance.xml = xml
instance.uuid = new_uuid
instance.save()
else:
# new submission
instance = Instance.objects.create(
xml=xml, user=user, status=status)
for f in media_files:
Attachment.objects.get_or_create(
instance=instance, media_file=f, mimetype=f.content_type)
# override date created if required
if date_created_override:
if not timezone.is_aware(date_created_override):
# default to utc?
date_created_override = timezone.make_aware(
date_created_override, timezone.utc)
instance.date_created = date_created_override
instance.save()
if instance.xform is not None:
pi, created = ParsedInstance.objects.get_or_create(
instance=instance)
if not created:
pi.save(async=False)
# commit all changes
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
return None
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is no username and a uuid, submitting a touchform.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
if username:
username = username.lower()
xml = xml_file.read()
is_touchform = False
# check alternative form submission ids
if not uuid:
# parse UUID from uploaded XML
split_xml = uuid_regex.split(xml)
# check that xml has UUID, then it is a crowdform
if len(split_xml) > 1:
uuid = split_xml[1]
else:
# is a touchform
is_touchform = True
if not username and not uuid:
raise InstanceInvalidUserError()
if uuid:
# try find the form by its uuid which is the ideal condition
if XForm.objects.filter(uuid=uuid).count() > 0:
xform = XForm.objects.get(uuid=uuid)
xform_username = xform.user.username
if xform_username != username and not xform.is_crowd_form \
and not is_touchform:
raise IsNotCrowdformError()
username = xform_username
# Else, since we have a username, the Instance creation logic will
# handle checking for the forms existence by its id_string
if username and request and request.user.is_authenticated():
id_string = get_id_string_from_xml_str(xml)
xform = XForm.objects.get(
id_string=id_string, user__username=username)
if not xform.is_crowd_form and not is_touchform \
and xform.user.profile.require_auth \
and xform.user != request.user:
raise PermissionDenied(
_(u"%(request_user)s is not allowed to make submissions "
u"to %(form_user)s's %(form_title)s form." % {
'request_user': request.user,
'form_user': xform.user,
'form_title': xform.title}))
user = get_object_or_404(User, username=username)
existing_instance_count = Instance.objects.filter(
xml=xml, user=user).count()
if existing_instance_count == 0:
proceed_to_create_instance = True
else:
existing_instance = Instance.objects.filter(xml=xml, user=user)[0]
if existing_instance.xform and\
not existing_instance.xform.has_start_time:
proceed_to_create_instance = True
else:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
proceed_to_create_instance = False
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
if proceed_to_create_instance:
# check if its an edit submission
old_uuid = get_deprecated_uuid_from_xml(xml)
instances = Instance.objects.filter(uuid=old_uuid)
if not date_created_override:
date_created_override = get_submission_date_from_xml(xml)
if instances:
instance = instances[0]
InstanceHistory.objects.create(
xml=instance.xml, xform_instance=instance, uuid=old_uuid)
instance.xml = xml
instance.uuid = new_uuid
instance.save()
else:
# new submission
instance = Instance.objects.create(
xml=xml, user=user, status=status)
for f in media_files:
Attachment.objects.get_or_create(
instance=instance, media_file=f, mimetype=f.content_type)
# override date created if required
if date_created_override:
if not timezone.is_aware(date_created_override):
# default to utc?
date_created_override = timezone.make_aware(
date_created_override, timezone.utc)
instance.date_created = date_created_override
instance.save()
if instance.xform is not None:
pi, created = ParsedInstance.objects.get_or_create(
instance=instance)
if not created:
pi.save(async=False)
# commit all changes
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
return None
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
with transaction.atomic():
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xml_hash = Instance.get_hash(xml)
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
# Duplicate instances are identified by identical content hash OR, when
# a content hash is not present, by string comparison of the full
# content
duplicate_instances = Instance.objects.filter(
Q(xml_hash=xml_hash) |
Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
)
try:
# Due to lazy QuerySet evaluation, the `filter()` above should not
# hit the database. This index retrieval is our single query
existing_instance = duplicate_instances[0]
except IndexError:
# No duplicate found
pass
else:
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and deprecated uuid's
new_uuid = get_uuid_from_xml(xml)
# TODO: Make sure `uuid` is indexed by the DB!
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
# ensure we have saved the extra attachments
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override)
return instance
if duplicate_instances:
# We are now outside the atomic block, so we can raise an exception
# without rolling back the extra attachments we created earlier
# NB: Since `ATOMIC_REQUESTS` is set at the database level, everything
# could still be rolled back if the calling view fails to handle an
# exception
raise DuplicateInstance()
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
* If there is a username and no uuid, submitting an old ODK form.
* If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated else None
if username:
username = username.lower()
xml = xml_file.read()
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
checksum = sha256(xml).hexdigest()
new_uuid = get_uuid_from_xml(xml)
filtered_instances = get_filtered_instances(
Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk)
existing_instance = get_first_record(filtered_instances.only('id'))
if existing_instance and \
(new_uuid or existing_instance.xform.has_start_time):
# ensure we have saved the extra attachments
with transaction.atomic():
save_attachments(xform, existing_instance, media_files)
existing_instance.save(update_fields=['json', 'date_modified'])
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
return DuplicateInstance()
# get new and deprecated UUIDs
history = InstanceHistory.objects.filter(
xform_instance__xform_id=xform.pk,
xform_instance__deleted_at__isnull=True,
uuid=new_uuid).only('xform_instance').first()
if history:
duplicate_instance = history.xform_instance
# ensure we have saved the extra attachments
with transaction.atomic():
save_attachments(xform, duplicate_instance, media_files)
duplicate_instance.save()
return DuplicateInstance()
try:
with transaction.atomic():
if isinstance(xml, bytes):
xml = xml.decode('utf-8')
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override, checksum)
except IntegrityError:
instance = get_first_record(Instance.objects.filter(
Q(checksum=checksum) | Q(uuid=new_uuid),
xform_id=xform.pk))
if instance:
attachment_names = [
a.media_file.name.split('/')[-1]
for a in Attachment.objects.filter(instance=instance)
]
media_files = [f for f in media_files
if f.name not in attachment_names]
save_attachments(xform, instance, media_files)
instance.save()
instance = DuplicateInstance()
return instance
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
* If there is a username and no uuid, submitting an old ODK form.
* If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
new_uuid = get_uuid_from_xml(xml)
filtered_instances = get_filtered_instances(Q(xml=xml) | Q(uuid=new_uuid),
xform_id=xform.pk)
existing_instance = filtered_instances.first()
if existing_instance and \
(new_uuid or existing_instance.xform.has_start_time):
# ensure we have saved the extra attachments
with transaction.atomic():
save_attachments(xform, existing_instance, media_files)
existing_instance.save(update_fields=['json', 'date_modified'])
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
return DuplicateInstance()
# get new and depracated uuid's
history = InstanceHistory.objects.filter(
xform_instance__xform_id=xform.pk,
uuid=new_uuid).only('xform_instance').first()
if history:
duplicate_instance = history.xform_instance
# ensure we have saved the extra attachments
with transaction.atomic():
save_attachments(xform, duplicate_instance, media_files)
duplicate_instance.save()
return DuplicateInstance()
try:
with transaction.atomic():
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override)
except IntegrityError:
instance = Instance.objects.filter(xml=xml, xform__id=xform.pk).first()
if instance:
attachment_names = [
a.media_file.name.split('/')[-1]
for a in Attachment.objects.filter(instance=instance)
]
for a in media_files:
if a.name in attachment_names:
media_files.remove(a)
save_attachments(xform, instance, media_files)
instance.save()
instance = DuplicateInstance()
return instance
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
* If there is a username and no uuid, submitting an old ODK form.
* If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated else None
if username:
username = username.lower()
xml = xml_file.read()
# print('x'*80)
# print(validate_data(xml))
if validate_data(xml):
pass
else:
raise FailedValidation()
xform = get_xform_from_submission(xml, username, uuid, request=request)
check_submission_permissions(request, xform)
checksum = sha256(xml).hexdigest()
new_uuid = get_uuid_from_xml(xml)
filtered_instances = get_filtered_instances(Q(checksum=checksum)
| Q(uuid=new_uuid),
xform_id=xform.pk)
existing_instance = get_first_record(filtered_instances.only('id'))
if existing_instance and \
(new_uuid or existing_instance.xform.has_start_time):
# ensure we have saved the extra attachments
with transaction.atomic():
save_attachments(xform,
existing_instance,
media_files,
remove_deleted_media=True)
existing_instance.save(update_fields=['json', 'date_modified'])
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
return DuplicateInstance()
# get new and deprecated UUIDs
history = InstanceHistory.objects.filter(
xform_instance__xform_id=xform.pk,
xform_instance__deleted_at__isnull=True,
uuid=new_uuid).only('xform_instance').first()
if history:
duplicate_instance = history.xform_instance
# ensure we have saved the extra attachments
with transaction.atomic():
save_attachments(xform,
duplicate_instance,
media_files,
remove_deleted_media=True)
duplicate_instance.save()
return DuplicateInstance()
try:
with transaction.atomic():
if isinstance(xml, bytes):
xml = xml.decode('utf-8')
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override, checksum,
request)
except IntegrityError:
instance = get_first_record(
Instance.objects.filter(Q(checksum=checksum) | Q(uuid=new_uuid),
xform_id=xform.pk))
if instance:
attachment_names = [
a.media_file.name.split('/')[-1]
for a in Attachment.objects.filter(instance=instance)
]
media_files = [
f for f in media_files if f.name not in attachment_names
]
save_attachments(xform, instance, media_files)
instance.save()
instance = DuplicateInstance()
return instance
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
with transaction.atomic():
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
#needs site later
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
existing_instance_count = Instance.objects.filter(
xml=xml, xform__user=xform.user).count()
if existing_instance_count > 0:
existing_instance = Instance.objects.filter(
xml=xml, xform__user=xform.user)[0]
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
# ensure we have saved the extra attachments
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override)
return instance
if duplicate_instances:
# We are now outside the atomic block, so we can raise an exception
# without rolling back the extra attachments we created earlier
# NB: Since `ATOMIC_REQUESTS` is set at the database level, everything
# could still be rolled back if the calling view fails to handle an
# exception
raise DuplicateInstance()
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
existing_instance_count = Instance.objects.filter(
xml=xml, xform__user=xform.user).count()
if existing_instance_count > 0:
existing_instance = Instance.objects.filter(
xml=xml, xform__user=xform.user)[0]
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f,
mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status, date_created_override)
# commit all changes
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def create_instance(username,
xml_file,
media_files,
status='submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xml_hash = Instance.get_hash(xml)
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
# get new and deprecated uuid's
new_uuid = get_uuid_from_xml(xml)
# Dorey's rule from 2012 (commit 890a67aa):
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
# The start-time requirement protected submissions with identical responses
# from being rejected as duplicates *before* KoBoCAT had the concept of
# submission UUIDs. Nowadays, OpenRosa requires clients to send a UUID (in
# `<instanceID>`) within every submission; if the incoming XML has a UUID
# and still exactly matches an existing submission, it's certainly a
# duplicate (https://docs.opendatakit.org/openrosa-metadata/#fields).
if xform.has_start_time or new_uuid is not None:
# XML matches are identified by identical content hash OR, when a
# content hash is not present, by string comparison of the full
# content, which is slow! Use the management command
# `populate_xml_hashes_for_instances` to hash existing submissions
existing_instance = Instance.objects.filter(
Q(xml_hash=xml_hash)
| Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
).first()
else:
existing_instance = None
if existing_instance:
# ensure we have saved the extra attachments
any_new_attachment = save_attachments(existing_instance, media_files)
if not any_new_attachment:
raise DuplicateInstance()
else:
# Update Mongo via the related ParsedInstance
existing_instance.parsed_instance.save(asynchronous=False)
return existing_instance
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status, date_created_override)
return instance
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
with transaction.atomic():
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xml_hash = Instance.get_hash(xml)
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
# Duplicate instances are identified by identical content hash OR, when
# a content hash is not present, by string comparison of the full
# content
duplicate_instances = Instance.objects.filter(
Q(xml_hash=xml_hash)
| Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
)
try:
# Due to lazy QuerySet evaluation, the `filter()` above should not
# hit the database. This index retrieval is our single query
existing_instance = duplicate_instances[0]
except IndexError:
# No duplicate found
pass
else:
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and deprecated uuid's
new_uuid = get_uuid_from_xml(xml)
# TODO: Make sure `uuid` is indexed by the DB!
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
# ensure we have saved the extra attachments
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f,
mimetype=f.content_type)
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override)
return instance
if duplicate_instances:
# We are now outside the atomic block, so we can raise an exception
# without rolling back the extra attachments we created earlier
# NB: Since `ATOMIC_REQUESTS` is set at the database level, everything
# could still be rolled back if the calling view fails to handle an
# exception
raise DuplicateInstance()
def create_new_submission(self, request, site, form):
fs_xf = FieldSightXF.objects.get(pk=form)
xform = fs_xf.xf
xml_file_list = self.request.FILES.pop('xml_submission_file', [])
xml_file = xml_file_list[0] if len(xml_file_list) else None
xml = xml_file.read()
username = self.kwargs.get('username')
user = get_object_or_404(User, username=username)
media_files = request.FILES.values()
new_uuid = get_uuid_from_xml(xml)
site_id = site
xml_hash = Instance.get_hash(xml)
if xform.has_start_time:
# XML matches are identified by identical content hash OR, when a
# content hash is not present, by string comparison of the full
# content, which is slow! Use the management command
# `populate_xml_hashes_for_instances` to hash existing submissions
existing_instance = Instance.objects.filter(
Q(xml_hash=xml_hash)
| Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml),
xform__user=xform.user,
).first()
else:
existing_instance = None
if existing_instance:
# ensure we have saved the extra attachments
any_new_attachment = save_attachments(existing_instance,
media_files)
if not any_new_attachment:
raise DuplicateInstance()
else:
context = self.get_serializer_context()
serializer = SubmissionSerializer(existing_instance,
context=context)
return Response(serializer.data,
headers=self.get_openrosa_headers(request),
status=status.HTTP_201_CREATED,
template_name=self.template_name)
with transaction.atomic():
if fs_xf.is_survey:
instance = save_submission(
xform=xform,
xml=xml,
media_files=media_files,
new_uuid=new_uuid,
submitted_by=user,
status='submitted_via_web',
date_created_override=None,
fxid=None,
site=None,
fs_poj_id=fs_xf.id,
project=fs_xf.project.id,
)
else:
if fs_xf.site:
instance = save_submission(
xform=xform,
xml=xml,
media_files=media_files,
new_uuid=new_uuid,
submitted_by=user,
status='submitted_via_web',
date_created_override=None,
fxid=fs_xf.id,
site=site_id,
)
else:
instance = save_submission(
xform=xform,
xml=xml,
media_files=media_files,
new_uuid=new_uuid,
submitted_by=user,
status='submitted_via_web',
date_created_override=None,
fxid=None,
site=site_id,
fs_poj_id=fs_xf.id,
project=fs_xf.project.id,
)
task_obj = CeleryTaskProgress.objects.create(
user=user,
description='Change site info',
task_type=25,
content_object=instance.fieldsight_instance)
if task_obj:
from onadata.apps.fieldsight.tasks import \
update_meta_details
update_meta_details.apply_async(
(fs_xf.id, instance.id, task_obj.id, site_id),
countdown=1)
else:
from onadata.apps.fieldsight.tasks import \
update_meta_details
update_meta_details.apply_async(
(fs_xf.id, instance.id, 0, site_id), countdown=1)
noti_type = 16
title = "new submission"
if instance.fieldsight_instance.site:
extra_object = instance.fieldsight_instance.site
extra_message = ""
project = extra_object.project
site = extra_object
organization = extra_object.project.organization
else:
extra_object = instance.fieldsight_instance.project
extra_message = "project"
project = extra_object
site = None
organization = extra_object.organization
instance.fieldsight_instance.logs.create(
source=user,
type=noti_type,
title=title,
organization=organization,
project=project,
site=site,
extra_object=extra_object,
extra_message=extra_message,
content_object=instance.fieldsight_instance)
context = self.get_serializer_context()
serializer = SubmissionSerializer(instance, context=context)
return Response(serializer.data,
headers=self.get_openrosa_headers(request),
status=status.HTTP_201_CREATED,
template_name=self.template_name)
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is a username and a uuid, submitting a new ODK form.
"""
with transaction.atomic():
instance = None
submitted_by = request.user \
if request and request.user.is_authenticated() else None
if username:
username = username.lower()
xml = xml_file.read()
xform = get_xform_from_submission(xml, username, uuid)
check_submission_permissions(request, xform)
existing_instance_count = Instance.objects.filter(
xml=xml, xform__user=xform.user).count()
if existing_instance_count > 0:
existing_instance = Instance.objects.filter(
xml=xml, xform__user=xform.user)[0]
if not existing_instance.xform or\
existing_instance.xform.has_start_time:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
# ensure we have saved the extra attachments
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
else:
instance = save_submission(xform, xml, media_files, new_uuid,
submitted_by, status,
date_created_override)
return instance
if duplicate_instances:
# We are now outside the atomic block, so we can raise an exception
# without rolling back the extra attachments we created earlier
# NB: Since `ATOMIC_REQUESTS` is set at the database level, everything
# could still be rolled back if the calling view fails to handle an
# exception
raise DuplicateInstance()
