def test_inject_instanceid(self):
"""
Test that 1 and only 1 instance id exists or is injected
"""
instance = Instance.objects.all().reverse()[0]
xml_str = self.__load_fixture("..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml")
# test that we dont have an instance id
uuid = get_uuid_from_xml(xml_str)
self.assertIsNone(uuid)
injected_xml_str = inject_instanceid(xml_str, instance.uuid)
# check that xml has the instanceid tag
uuid = get_uuid_from_xml(injected_xml_str)
self.assertEqual(uuid, instance.uuid)
def test_inject_instanceid(self):
"""
Test that 1 and only 1 instance id exists or is injected
"""
instance = Instance.objects.all().reverse()[0]
xml_str = self.__load_fixture("..", "fixtures", "tutorial",
"instances",
"tutorial_2012-06-27_11-27-53.xml")
# test that we dont have an instance id
uuid = get_uuid_from_xml(xml_str)
self.assertIsNone(uuid)
injected_xml_str = inject_instanceid(xml_str, instance.uuid)
# check that xml has the instanceid tag
uuid = get_uuid_from_xml(injected_xml_str)
self.assertEqual(uuid, instance.uuid)
def inject_instanceid(xml_str, uuid):
if get_uuid_from_xml(xml_str) is None:
xml = clean_and_parse_xml(xml_str)
children = xml.childNodes
if children.length == 0:
raise ValueError(_("XML string must have a survey element."))
# check if we have a meta tag
survey_node = children.item(0)
meta_tags = [
n for n in survey_node.childNodes
if n.nodeType == Node.ELEMENT_NODE
and n.tagName.lower() == "meta"]
if len(meta_tags) == 0:
meta_tag = xml.createElement("meta")
xml.documentElement.appendChild(meta_tag)
else:
meta_tag = meta_tags[0]
# check if we have an instanceID tag
uuid_tags = [
n for n in meta_tag.childNodes
if n.nodeType == Node.ELEMENT_NODE
and n.tagName == "instanceID"]
if len(uuid_tags) == 0:
uuid_tag = xml.createElement("instanceID")
meta_tag.appendChild(uuid_tag)
else:
uuid_tag = uuid_tags[0]
# insert meta and instanceID
text_node = xml.createTextNode(u"uuid:%s" % uuid)
uuid_tag.appendChild(text_node)
return xml.toxml()
return xml_str
def inject_instanceid(xml_str, uuid):
if get_uuid_from_xml(xml_str) is None:
xml = clean_and_parse_xml(xml_str)
children = xml.childNodes
if children.length == 0:
raise ValueError(_("XML string must have a survey element."))
# check if we have a meta tag
survey_node = children.item(0)
meta_tags = [
n for n in survey_node.childNodes
if n.nodeType == Node.ELEMENT_NODE and n.tagName.lower() == "meta"
]
if len(meta_tags) == 0:
meta_tag = xml.createElement("meta")
xml.documentElement.appendChild(meta_tag)
else:
meta_tag = meta_tags[0]
# check if we have an instanceID tag
uuid_tags = [
n for n in meta_tag.childNodes
if n.nodeType == Node.ELEMENT_NODE and n.tagName == "instanceID"
]
if len(uuid_tags) == 0:
uuid_tag = xml.createElement("instanceID")
meta_tag.appendChild(uuid_tag)
else:
uuid_tag = uuid_tags[0]
# insert meta and instanceID
text_node = xml.createTextNode(u"uuid:%s" % uuid)
uuid_tag.appendChild(text_node)
return xml.toxml()
return xml_str
def test_inject_instanceid(self):
"""
Test that 1 and only 1 instance id exists or is injected
"""
instance = Instance.objects.all().reverse()[0]
with open(
os.path.join(
os.path.dirname(__file__), "..", "fixtures", "tutorial",
"instances", "tutorial_2012-06-27_11-27-53.xml"),
"r") as xml_file:
xml_str = xml_file.read()
# test that we dont have an instance id
uuid = get_uuid_from_xml(xml_str)
self.assertIsNone(uuid)
injected_xml_str = inject_instanceid(xml_str, instance.uuid)
# check that xml has the instanceid tag
uuid = get_uuid_from_xml(injected_xml_str)
self.assertEqual(uuid, instance.uuid)
def test_get_uuid_from_xml(self):
with open(
os.path.join(
os.path.dirname(__file__), "..", "fixtures", "tutorial",
"instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml"),
"r") as xml_file:
xml_str = xml_file.read()
instanceID = get_uuid_from_xml(xml_str)
self.assertEqual(instanceID, "729f173c688e482486a48661700455ff")
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is no username and a uuid, submitting a touchform.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
if username:
username = username.lower()
xml = xml_file.read()
is_touchform = False
# check alternative form submission ids
if not uuid:
# parse UUID from uploaded XML
split_xml = uuid_regex.split(xml)
# check that xml has UUID, then it is a crowdform
if len(split_xml) > 1:
uuid = split_xml[1]
else:
# is a touchform
is_touchform = True
if not username and not uuid:
raise InstanceInvalidUserError()
if uuid:
# try find the fomr by its uuid which is the ideal condition
if XForm.objects.filter(uuid=uuid).count() > 0:
xform = XForm.objects.get(uuid=uuid)
xform_username = xform.user.username
if xform_username != username and not xform.is_crowd_form \
and not is_touchform:
raise IsNotCrowdformError()
username = xform_username
# else, since we have a username, the Instance creation logic will
# handle checking for the forms existence by its id_string
user = get_object_or_404(User, username=username)
existing_instance_count = Instance.objects.filter(
xml=xml, user=user).count()
if existing_instance_count == 0:
proceed_to_create_instance = True
else:
existing_instance = Instance.objects.filter(xml=xml, user=user)[0]
if existing_instance.xform and\
not existing_instance.xform.has_start_time:
proceed_to_create_instance = True
else:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
proceed_to_create_instance = False
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f, mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
if proceed_to_create_instance:
# check if its an edit submission
old_uuid = get_deprecated_uuid_from_xml(xml)
instances = Instance.objects.filter(uuid=old_uuid)
if instances:
instance = instances[0]
InstanceHistory.objects.create(
xml=instance.xml, xform_instance=instance, uuid=old_uuid)
instance.xml = xml
instance.uuid = new_uuid
instance.save()
else:
# new submission
instance = Instance.objects.create(
xml=xml, user=user, status=status)
for f in media_files:
Attachment.objects.get_or_create(
instance=instance, media_file=f, mimetype=f.content_type)
# override date created if required
if date_created_override:
instance.date_created = date_created_override
instance.save()
if instance.xform is not None:
pi, created = ParsedInstance.objects.get_or_create(
instance=instance)
if not created:
pi.save(async=False)
# commit all changes
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
return None
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is no username and a uuid, submitting a touchform.
If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
xform = None
if username:
username = username.lower()
xml = xml_file.read()
is_touchform = False
# check alternative form submission ids
if not uuid:
# parse UUID from uploaded XML
split_xml = uuid_regex.split(xml)
# check that xml has UUID, then it is a crowdform
if len(split_xml) > 1:
uuid = split_xml[1]
else:
# is a touchform
is_touchform = True
if not username and not uuid:
raise InstanceInvalidUserError()
if uuid:
# try find the form by its uuid which is the ideal condition
try:
xform = XForm.objects.get(uuid=uuid)
xform_username = xform.user.username
if xform_username != username and not xform.is_crowd_form \
and not is_touchform:
raise IsNotCrowdformError()
username = xform_username
except XForm.DoesNotExist:
pass
# else, since we have a username, the Instance creation logic will
# handle checking for the forms existence by its id_string
if username and request and request.user.is_authenticated():
id_string = get_id_string_from_xml_str(xml)
if xform is None:
try:
xform = XForm.objects.get(id_string=id_string,
user__username=username)
except XForm.DoesNotExist:
pass
if xform is not None:
if not xform.is_crowd_form and not is_touchform \
and xform.user.profile.require_auth \
and xform.user != request.user:
raise PermissionDenied(
_(
u"%(request_user)s is not allowed to make submissions "
u"to %(form_user)s's %(form_title)s form." % {
'request_user': request.user,
'form_user': xform.user,
'form_title': xform.title
}))
user = get_object_or_404(User, username=username)
try:
# make sure the Instance object doesn't already exist
# using good db optimization practices
# (https://docs.djangoproject.com/en/1.5/topics/db/optimization/),
# there is no need to fetch the whole Instance object, when all
# we need to check for is existence and an attribute in the associated XForm object
# this code is logically-equivalent to, but more efficient than, the legacy code below it
xform_pk = Instance.objects.filter(xml=xml,
user=user).values_list('xform',
flat=True)[0]
if XForm.objects.filter(pk=xform_pk).values_list('has_start_time',
flat=True)[0]:
raise DuplicateInstance()
# don't do it this way, kids, especially with 2.8 million Instances and counting...
#existing_instance = Instance.objects.filter(xml=xml, user=user)[0]
#if existing_instance.xform and existing_instance.xform.has_start_time:
#raise DuplicateInstance()
except IndexError:
pass
# get new and deprecated uuids
new_uuid = get_uuid_from_xml(xml)
if new_uuid is not None:
# this xml is a duplicate, however, there may be
# new attachments, so save them
try:
# here, too, all we need is the pk so don't retrieve the whole object
duplicate_instance_pk = Instance.objects.filter(
uuid=new_uuid).values_list('pk', flat=True)[0]
if media_files:
_save_attachments.delay(duplicate_instance_pk, media_files)
raise DuplicateInstance()
except IndexError:
pass
# proceed_to_create_instance = True (as per legacy logic)
# check if its an edit submission
old_uuid = get_deprecated_uuid_from_xml(xml)
if old_uuid is not None:
try:
# here is a case where we do need the whole instance, so stet this code
instance = Instance.objects.filter(uuid=old_uuid)[0]
except IndexError:
pass
if not date_created_override:
date_created_override = get_submission_date_from_xml(xml)
if instance is not None:
# this is an edit
InstanceHistory.objects.create(xml=instance.xml,
xform_instance=instance,
uuid=old_uuid)
instance.xml = xml
instance.uuid = new_uuid
else:
# new submission
instance = Instance.objects.create(xml=xml, user=user, status=status)
# override date created if required
if date_created_override:
if not timezone.is_aware(date_created_override):
# default to utc?
date_created_override = timezone.make_aware(
date_created_override, timezone.utc)
instance.date_created = date_created_override
instance.save()
if instance.xform is not None:
pi, created = ParsedInstance.objects.get_or_create(instance=instance)
if not created:
pi.save(async=False)
if media_files:
_save_attachments.delay(instance.pk, media_files)
return instance
def create_instance(username,
xml_file,
media_files,
status=u'submitted_via_web',
uuid=None,
date_created_override=None,
request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is no username and a uuid, submitting a touchform.
If there is a username and a uuid, submitting a new ODK form.
"""
try:
if username:
username = username.lower()
xml = xml_file.read()
is_touchform = False
# check alternative form submission ids
if not uuid:
# parse UUID from uploaded XML
split_xml = uuid_regex.split(xml)
# check that xml has UUID, then it is a crowdform
if len(split_xml) > 1:
uuid = split_xml[1]
else:
# is a touchform
is_touchform = True
if not username and not uuid:
raise InstanceInvalidUserError()
if uuid:
# try find the fomr by its uuid which is the ideal condition
if XForm.objects.filter(uuid=uuid).count() > 0:
xform = XForm.objects.get(uuid=uuid)
xform_username = xform.user.username
if xform_username != username and not xform.is_crowd_form \
and not is_touchform:
raise IsNotCrowdformError()
username = xform_username
# else, since we have a username, the Instance creation logic will
# handle checking for the forms existence by its id_string
if username and request and request.user.is_authenticated():
id_string = get_id_string_from_xml_str(xml)
xform = XForm.objects.get(id_string=id_string,
user__username=username)
if not xform.is_crowd_form and not is_touchform \
and xform.user.profile.require_auth \
and xform.user != request.user:
raise PermissionDenied(
_(
u"%(request_user)s is not allowed to make submissions "
u"to %(form_user)s's %(form_title)s form." % {
'request_user': request.user,
'form_user': xform.user,
'form_title': xform.title
}))
user = get_object_or_404(User, username=username)
existing_instance_count = Instance.objects.filter(xml=xml,
user=user).count()
if existing_instance_count == 0:
proceed_to_create_instance = True
else:
existing_instance = Instance.objects.filter(xml=xml, user=user)[0]
if existing_instance.xform and\
not existing_instance.xform.has_start_time:
proceed_to_create_instance = True
else:
# Ignore submission as a duplicate IFF
# * a submission's XForm collects start time
# * the submitted XML is an exact match with one that
# has already been submitted for that user.
proceed_to_create_instance = False
raise DuplicateInstance()
# get new and depracated uuid's
new_uuid = get_uuid_from_xml(xml)
duplicate_instances = Instance.objects.filter(uuid=new_uuid)
if duplicate_instances:
for f in media_files:
Attachment.objects.get_or_create(
instance=duplicate_instances[0],
media_file=f,
mimetype=f.content_type)
# ensure we have saved the extra attachments
transaction.commit()
raise DuplicateInstance()
if proceed_to_create_instance:
# check if its an edit submission
old_uuid = get_deprecated_uuid_from_xml(xml)
instances = Instance.objects.filter(uuid=old_uuid)
if not date_created_override:
date_created_override = get_submission_date_from_xml(xml)
if instances:
instance = instances[0]
InstanceHistory.objects.create(xml=instance.xml,
xform_instance=instance,
uuid=old_uuid)
instance.xml = xml
instance.uuid = new_uuid
instance.save()
else:
# new submission
instance = Instance.objects.create(xml=xml,
user=user,
status=status)
for f in media_files:
Attachment.objects.get_or_create(instance=instance,
media_file=f,
mimetype=f.content_type)
# override date created if required
if date_created_override:
if not timezone.is_aware(date_created_override):
# default to utc?
date_created_override = timezone.make_aware(
date_created_override, timezone.utc)
instance.date_created = date_created_override
instance.save()
if instance.xform is not None:
pi, created = ParsedInstance.objects.get_or_create(
instance=instance)
if not created:
pi.save(async=False)
# commit all changes
transaction.commit()
return instance
except Exception:
transaction.rollback()
raise
return None
def create_instance(username, xml_file, media_files,
status=u'submitted_via_web', uuid=None,
date_created_override=None, request=None):
"""
I used to check if this file had been submitted already, I've
taken this out because it was too slow. Now we're going to create
a way for an admin to mark duplicate instances. This should
simplify things a bit.
Submission cases:
If there is a username and no uuid, submitting an old ODK form.
If there is no username and a uuid, submitting a touchform.
If there is a username and a uuid, submitting a new ODK form.
"""
instance = None
xform = None
if username:
username = username.lower()
xml = xml_file.read()
is_touchform = False
# check alternative form submission ids
if not uuid:
# parse UUID from uploaded XML
split_xml = uuid_regex.split(xml)
# check that xml has UUID, then it is a crowdform
if len(split_xml) > 1:
uuid = split_xml[1]
else:
# is a touchform
is_touchform = True
if not username and not uuid:
raise InstanceInvalidUserError()
if uuid:
# try find the form by its uuid which is the ideal condition
try:
xform = XForm.objects.get(uuid=uuid)
xform_username = xform.user.username
if xform_username != username and not xform.is_crowd_form \
and not is_touchform:
raise IsNotCrowdformError()
username = xform_username
except XForm.DoesNotExist:
pass
# else, since we have a username, the Instance creation logic will
# handle checking for the forms existence by its id_string
if username and request and request.user.is_authenticated():
id_string = get_id_string_from_xml_str(xml)
if xform is None:
try:
xform = XForm.objects.get(id_string=id_string, user__username=username)
except XForm.DoesNotExist:
pass
if xform is not None:
if not xform.is_crowd_form and not is_touchform \
and xform.user.profile.require_auth \
and xform.user != request.user:
raise PermissionDenied(
_(u"%(request_user)s is not allowed to make submissions "
u"to %(form_user)s's %(form_title)s form." % {
'request_user': request.user,
'form_user': xform.user,
'form_title': xform.title}))
user = get_object_or_404(User, username=username)
try:
# make sure the Instance object doesn't already exist
# using good db optimization practices
# (https://docs.djangoproject.com/en/1.5/topics/db/optimization/),
# there is no need to fetch the whole Instance object, when all
# we need to check for is existence and an attribute in the associated XForm object
# this code is logically-equivalent to, but more efficient than, the legacy code below it
xform_pk = Instance.objects.filter(xml=xml, user=user).values_list('xform', flat=True)[0]
if XForm.objects.filter(pk=xform_pk).values_list('has_start_time', flat=True)[0]:
raise DuplicateInstance()
# don't do it this way, kids, especially with 2.8 million Instances and counting...
#existing_instance = Instance.objects.filter(xml=xml, user=user)[0]
#if existing_instance.xform and existing_instance.xform.has_start_time:
#raise DuplicateInstance()
except IndexError:
pass
# get new and deprecated uuids
new_uuid = get_uuid_from_xml(xml)
if new_uuid is not None:
# this xml is a duplicate, however, there may be
# new attachments, so save them
try:
# here, too, all we need is the pk so don't retrieve the whole object
duplicate_instance_pk = Instance.objects.filter(uuid=new_uuid).values_list('pk', flat=True)[0]
if media_files:
_save_attachments.delay(duplicate_instance_pk, media_files)
raise DuplicateInstance()
except IndexError:
pass
# proceed_to_create_instance = True (as per legacy logic)
# check if its an edit submission
old_uuid = get_deprecated_uuid_from_xml(xml)
if old_uuid is not None:
try:
# here is a case where we do need the whole instance, so stet this code
instance = Instance.objects.filter(uuid=old_uuid)[0]
except IndexError:
pass
if not date_created_override:
date_created_override = get_submission_date_from_xml(xml)
if instance is not None:
# this is an edit
InstanceHistory.objects.create(
xml=instance.xml, xform_instance=instance, uuid=old_uuid)
instance.xml = xml
instance.uuid = new_uuid
else:
# new submission
instance = Instance.objects.create(
xml=xml, user=user, status=status)
# override date created if required
if date_created_override:
if not timezone.is_aware(date_created_override):
# default to utc?
date_created_override = timezone.make_aware(
date_created_override, timezone.utc)
instance.date_created = date_created_override
instance.save()
if instance.xform is not None:
pi, created = ParsedInstance.objects.get_or_create(
instance=instance)
if not created:
pi.save(async=False)
if media_files:
_save_attachments.delay(instance.pk, media_files)
return instance
