def test_inject_instanceid(self): """ Test that 1 and only 1 instance id exists or is injected """ instance = Instance.objects.all().reverse()[0] xml_str = self.__load_fixture("..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml") # test that we dont have an instance id uuid = get_uuid_from_xml(xml_str) self.assertIsNone(uuid) injected_xml_str = inject_instanceid(xml_str, instance.uuid) # check that xml has the instanceid tag uuid = get_uuid_from_xml(injected_xml_str) self.assertEqual(uuid, instance.uuid)
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ try: instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) existing_instance_count = Instance.objects.filter( xml=xml, xform__user=xform.user).count() if existing_instance_count > 0: existing_instance = Instance.objects.filter( xml=xml, xform__user=xform.user)[0] if not existing_instance.xform or\ existing_instance.xform.has_start_time: # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. raise DuplicateInstance() # get new and depracated uuid's new_uuid = get_uuid_from_xml(xml) duplicate_instances = Instance.objects.filter(uuid=new_uuid) if duplicate_instances: for f in media_files: Attachment.objects.get_or_create( instance=duplicate_instances[0], media_file=f, mimetype=f.content_type) # ensure we have saved the extra attachments transaction.commit() raise DuplicateInstance() instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) # commit all changes transaction.commit() return instance except Exception: transaction.rollback() raise
def inject_instanceid(xml_str, uuid): if get_uuid_from_xml(xml_str) is None: xml = clean_and_parse_xml(xml_str) children = xml.childNodes if children.length == 0: raise ValueError(_("XML string must have a survey element.")) # check if we have a meta tag survey_node = children.item(0) meta_tags = [ n for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName.lower() == "meta"] if len(meta_tags) == 0: meta_tag = xml.createElement("meta") xml.documentElement.appendChild(meta_tag) else: meta_tag = meta_tags[0] # check if we have an instanceID tag uuid_tags = [ n for n in meta_tag.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "instanceID"] if len(uuid_tags) == 0: uuid_tag = xml.createElement("instanceID") meta_tag.appendChild(uuid_tag) else: uuid_tag = uuid_tags[0] # insert meta and instanceID text_node = xml.createTextNode(u"uuid:%s" % uuid) uuid_tag.appendChild(text_node) return xml.toxml() return xml_str
def inject_instanceid(xml_str, uuid): if get_uuid_from_xml(xml_str) is None: xml = clean_and_parse_xml(xml_str) children = xml.childNodes if children.length == 0: raise ValueError(_("XML string must have a survey element.")) # check if we have a meta tag survey_node = children.item(0) meta_tags = [ n for n in survey_node.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName.lower() == "meta" ] if len(meta_tags) == 0: meta_tag = xml.createElement("meta") xml.documentElement.appendChild(meta_tag) else: meta_tag = meta_tags[0] # check if we have an instanceID tag uuid_tags = [ n for n in meta_tag.childNodes if n.nodeType == Node.ELEMENT_NODE and n.tagName == "instanceID" ] if len(uuid_tags) == 0: uuid_tag = xml.createElement("instanceID") meta_tag.appendChild(uuid_tag) else: uuid_tag = uuid_tags[0] # insert meta and instanceID text_node = xml.createTextNode(u"uuid:%s" % uuid) uuid_tag.appendChild(text_node) return xml.toxml() return xml_str
def _set_uuid(self): # pylint: disable=no-member, attribute-defined-outside-init if self.xml and not self.uuid: # pylint: disable=no-member uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self)
def _set_uuid(self): # pylint: disable=E1101, E0203 if self.xml and not self.uuid: # pylint: disable=E1101 uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self)
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() xml_hash = Instance.get_hash(xml) xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) # Dorey's rule from 2012 (commit 890a67aa): # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. if xform.has_start_time: # XML matches are identified by identical content hash OR, when a # content hash is not present, by string comparison of the full # content, which is slow! Use the management command # `populate_xml_hashes_for_instances` to hash existing submissions existing_instance = Instance.objects.filter( Q(xml_hash=xml_hash) | Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml), xform__user=xform.user, ).first() else: existing_instance = None # get new and deprecated uuid's new_uuid = get_uuid_from_xml(xml) if existing_instance: # ensure we have saved the extra attachments any_new_attachment = save_attachments(existing_instance, media_files) if not any_new_attachment: raise DuplicateInstance() else: # Update Mongo via the related ParsedInstance existing_instance.parsed_instance.save(async=False) return existing_instance else: instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) return instance
def test_get_uuid_from_xml(self): with open( os.path.join(os.path.dirname(__file__), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml"), "r") as xml_file: xml_str = xml_file.read() instanceID = get_uuid_from_xml(xml_str) self.assertEqual(instanceID, "729f173c688e482486a48661700455ff")
def test_get_uuid_from_xml(self): with open( os.path.join( os.path.dirname(__file__), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml"), "r") as xml_file: xml_str = xml_file.read() instanceID = get_uuid_from_xml(xml_str) self.assertEqual(instanceID, "729f173c688e482486a48661700455ff")
def create_instance(fsxfid, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None, site=None, fs_proj_xf=None, proj_id=None, xform=None, flagged_instance=None): with transaction.atomic(): instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None xml = xml_file.read() xml_hash = Instance.get_hash(xml) if xform.has_start_time: # XML matches are identified by identical content hash OR, when a # content hash is not present, by string comparison of the full # content, which is slow! Use the management command # `populate_xml_hashes_for_instances` to hash existing submissions existing_instance = Instance.objects.filter( Q(xml_hash=xml_hash) | Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml), xform__user=xform.user, ).first() else: existing_instance = None new_uuid = get_uuid_from_xml(xml) uuid = new_uuid if existing_instance: # ensure we have saved the extra attachments any_new_attachment = save_attachments(existing_instance, media_files) if not any_new_attachment: raise DuplicateInstance() else: # Update Mongo via the related ParsedInstance if fs_proj_xf: fs_poj_id = str(fs_proj_xf) else: fs_poj_id = "" pi, created = FieldSightParsedInstance.get_or_create(existing_instance, update_data={'fs_uuid': str(fsxfid), 'fs_status': 0, 'fs_site': site, 'fs_project': proj_id, 'fs_project_uuid': fs_poj_id}) print() return existing_instance else: if fsxfid is None: fsxfid = "" if site is None: site = "" instance = save_submission(xform, xml, media_files, uuid, submitted_by, status, date_created_override, str(fsxfid), str(site), fs_proj_xf, proj_id, flagged_instance) return instance
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ try: instance = None user = get_object_or_404(User, username=username.lower()) submitted_by = request.user \ if request and request.user.is_authenticated() else user if username: username = username.lower() xml = xml_file.read() #print 'xml of form'+str(xml) #valid = check_custom_form_validation(xml,request,username) valid = True if not valid: #print 'Form is not Valid..... ' transaction.rollback() raise ValueError('need to register your tubewell first...') xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform, user) existing_instance_count = Instance.objects.filter( xml=xml, xform__user=xform.user).count() if existing_instance_count > 0: existing_instance = Instance.objects.filter( xml=xml, xform__user=xform.user)[0] if not existing_instance.xform or\ existing_instance.xform.has_start_time: # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. raise DuplicateInstance() # get new and depracated uuid's new_uuid = get_uuid_from_xml(xml) duplicate_instances = Instance.objects.filter(uuid=new_uuid) if duplicate_instances: for f in media_files: Attachment.objects.get_or_create( instance=duplicate_instances[0], media_file=f, mimetype=f.content_type) # ensure we have saved the extra attachments transaction.commit() raise DuplicateInstance() if is_exist_farmer_paravet_ait(xml, request, username, user) == 1: transaction.commit() raise DuplicateFarmer() instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) # commit all changes call_parave_ai_reg_api(xml, request, username, user) send_push_noti_after_profile_update(xml) transaction.commit() return instance except Exception: transaction.rollback() raise
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is no username and a uuid, submitting a touchform. If there is a username and a uuid, submitting a new ODK form. """ try: if username: username = username.lower() xml = xml_file.read() is_touchform = False # check alternative form submission ids if not uuid: # parse UUID from uploaded XML split_xml = uuid_regex.split(xml) # check that xml has UUID, then it is a crowdform if len(split_xml) > 1: uuid = split_xml[1] else: # is a touchform is_touchform = True if not username and not uuid: raise InstanceInvalidUserError() if uuid: # try find the form by its uuid which is the ideal condition if XForm.objects.filter(uuid=uuid).count() > 0: xform = XForm.objects.get(uuid=uuid) xform_username = xform.user.username if xform_username != username and not xform.is_crowd_form \ and not is_touchform: raise IsNotCrowdformError() username = xform_username # Else, since we have a username, the Instance creation logic will # handle checking for the forms existence by its id_string if username and request and request.user.is_authenticated(): id_string = get_id_string_from_xml_str(xml) xform = XForm.objects.get( id_string=id_string, user__username=username) if not xform.is_crowd_form and not is_touchform \ and xform.user.profile.require_auth \ and xform.user != request.user: raise PermissionDenied( _(u"%(request_user)s is not allowed to make submissions " u"to %(form_user)s's %(form_title)s form." % { 'request_user': request.user, 'form_user': xform.user, 'form_title': xform.title})) user = get_object_or_404(User, username=username) existing_instance_count = Instance.objects.filter( xml=xml, user=user).count() if existing_instance_count == 0: proceed_to_create_instance = True else: existing_instance = Instance.objects.filter(xml=xml, user=user)[0] if existing_instance.xform and\ not existing_instance.xform.has_start_time: proceed_to_create_instance = True else: # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. proceed_to_create_instance = False raise DuplicateInstance() # get new and depracated uuid's new_uuid = get_uuid_from_xml(xml) duplicate_instances = Instance.objects.filter(uuid=new_uuid) if duplicate_instances: for f in media_files: Attachment.objects.get_or_create( instance=duplicate_instances[0], media_file=f, mimetype=f.content_type) # ensure we have saved the extra attachments transaction.commit() raise DuplicateInstance() if proceed_to_create_instance: # check if its an edit submission old_uuid = get_deprecated_uuid_from_xml(xml) instances = Instance.objects.filter(uuid=old_uuid) if not date_created_override: date_created_override = get_submission_date_from_xml(xml) if instances: instance = instances[0] InstanceHistory.objects.create( xml=instance.xml, xform_instance=instance, uuid=old_uuid) instance.xml = xml instance.uuid = new_uuid instance.save() else: # new submission instance = Instance.objects.create( xml=xml, user=user, status=status) for f in media_files: Attachment.objects.get_or_create( instance=instance, media_file=f, mimetype=f.content_type) # override date created if required if date_created_override: if not timezone.is_aware(date_created_override): # default to utc? date_created_override = timezone.make_aware( date_created_override, timezone.utc) instance.date_created = date_created_override instance.save() if instance.xform is not None: pi, created = ParsedInstance.objects.get_or_create( instance=instance) if not created: pi.save(async=False) # commit all changes transaction.commit() return instance except Exception: transaction.rollback() raise return None
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ with transaction.atomic(): instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() xml_hash = Instance.get_hash(xml) xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) # Duplicate instances are identified by identical content hash OR, when # a content hash is not present, by string comparison of the full # content duplicate_instances = Instance.objects.filter( Q(xml_hash=xml_hash) | Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml), xform__user=xform.user, ) try: # Due to lazy QuerySet evaluation, the `filter()` above should not # hit the database. This index retrieval is our single query existing_instance = duplicate_instances[0] except IndexError: # No duplicate found pass else: if not existing_instance.xform or\ existing_instance.xform.has_start_time: # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. raise DuplicateInstance() # get new and deprecated uuid's new_uuid = get_uuid_from_xml(xml) # TODO: Make sure `uuid` is indexed by the DB! duplicate_instances = Instance.objects.filter(uuid=new_uuid) if duplicate_instances: # ensure we have saved the extra attachments for f in media_files: Attachment.objects.get_or_create( instance=duplicate_instances[0], media_file=f, mimetype=f.content_type) else: instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) return instance if duplicate_instances: # We are now outside the atomic block, so we can raise an exception # without rolling back the extra attachments we created earlier # NB: Since `ATOMIC_REQUESTS` is set at the database level, everything # could still be rolled back if the calling view fails to handle an # exception raise DuplicateInstance()
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: * If there is a username and no uuid, submitting an old ODK form. * If there is a username and a uuid, submitting a new ODK form. """ instance = None submitted_by = request.user \ if request and request.user.is_authenticated else None if username: username = username.lower() xml = xml_file.read() xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) checksum = sha256(xml).hexdigest() new_uuid = get_uuid_from_xml(xml) filtered_instances = get_filtered_instances( Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk) existing_instance = get_first_record(filtered_instances.only('id')) if existing_instance and \ (new_uuid or existing_instance.xform.has_start_time): # ensure we have saved the extra attachments with transaction.atomic(): save_attachments(xform, existing_instance, media_files) existing_instance.save(update_fields=['json', 'date_modified']) # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. return DuplicateInstance() # get new and deprecated UUIDs history = InstanceHistory.objects.filter( xform_instance__xform_id=xform.pk, xform_instance__deleted_at__isnull=True, uuid=new_uuid).only('xform_instance').first() if history: duplicate_instance = history.xform_instance # ensure we have saved the extra attachments with transaction.atomic(): save_attachments(xform, duplicate_instance, media_files) duplicate_instance.save() return DuplicateInstance() try: with transaction.atomic(): if isinstance(xml, bytes): xml = xml.decode('utf-8') instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override, checksum) except IntegrityError: instance = get_first_record(Instance.objects.filter( Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk)) if instance: attachment_names = [ a.media_file.name.split('/')[-1] for a in Attachment.objects.filter(instance=instance) ] media_files = [f for f in media_files if f.name not in attachment_names] save_attachments(xform, instance, media_files) instance.save() instance = DuplicateInstance() return instance
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: * If there is a username and no uuid, submitting an old ODK form. * If there is a username and a uuid, submitting a new ODK form. """ instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) new_uuid = get_uuid_from_xml(xml) filtered_instances = get_filtered_instances(Q(xml=xml) | Q(uuid=new_uuid), xform_id=xform.pk) existing_instance = filtered_instances.first() if existing_instance and \ (new_uuid or existing_instance.xform.has_start_time): # ensure we have saved the extra attachments with transaction.atomic(): save_attachments(xform, existing_instance, media_files) existing_instance.save(update_fields=['json', 'date_modified']) # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. return DuplicateInstance() # get new and depracated uuid's history = InstanceHistory.objects.filter( xform_instance__xform_id=xform.pk, uuid=new_uuid).only('xform_instance').first() if history: duplicate_instance = history.xform_instance # ensure we have saved the extra attachments with transaction.atomic(): save_attachments(xform, duplicate_instance, media_files) duplicate_instance.save() return DuplicateInstance() try: with transaction.atomic(): instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) except IntegrityError: instance = Instance.objects.filter(xml=xml, xform__id=xform.pk).first() if instance: attachment_names = [ a.media_file.name.split('/')[-1] for a in Attachment.objects.filter(instance=instance) ] for a in media_files: if a.name in attachment_names: media_files.remove(a) save_attachments(xform, instance, media_files) instance.save() instance = DuplicateInstance() return instance
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: * If there is a username and no uuid, submitting an old ODK form. * If there is a username and a uuid, submitting a new ODK form. """ instance = None submitted_by = request.user \ if request and request.user.is_authenticated else None if username: username = username.lower() xml = xml_file.read() # print('x'*80) # print(validate_data(xml)) if validate_data(xml): pass else: raise FailedValidation() xform = get_xform_from_submission(xml, username, uuid, request=request) check_submission_permissions(request, xform) checksum = sha256(xml).hexdigest() new_uuid = get_uuid_from_xml(xml) filtered_instances = get_filtered_instances(Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk) existing_instance = get_first_record(filtered_instances.only('id')) if existing_instance and \ (new_uuid or existing_instance.xform.has_start_time): # ensure we have saved the extra attachments with transaction.atomic(): save_attachments(xform, existing_instance, media_files, remove_deleted_media=True) existing_instance.save(update_fields=['json', 'date_modified']) # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. return DuplicateInstance() # get new and deprecated UUIDs history = InstanceHistory.objects.filter( xform_instance__xform_id=xform.pk, xform_instance__deleted_at__isnull=True, uuid=new_uuid).only('xform_instance').first() if history: duplicate_instance = history.xform_instance # ensure we have saved the extra attachments with transaction.atomic(): save_attachments(xform, duplicate_instance, media_files, remove_deleted_media=True) duplicate_instance.save() return DuplicateInstance() try: with transaction.atomic(): if isinstance(xml, bytes): xml = xml.decode('utf-8') instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override, checksum, request) except IntegrityError: instance = get_first_record( Instance.objects.filter(Q(checksum=checksum) | Q(uuid=new_uuid), xform_id=xform.pk)) if instance: attachment_names = [ a.media_file.name.split('/')[-1] for a in Attachment.objects.filter(instance=instance) ] media_files = [ f for f in media_files if f.name not in attachment_names ] save_attachments(xform, instance, media_files) instance.save() instance = DuplicateInstance() return instance
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ with transaction.atomic(): instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() #needs site later xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) existing_instance_count = Instance.objects.filter( xml=xml, xform__user=xform.user).count() if existing_instance_count > 0: existing_instance = Instance.objects.filter( xml=xml, xform__user=xform.user)[0] if not existing_instance.xform or\ existing_instance.xform.has_start_time: # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. raise DuplicateInstance() # get new and depracated uuid's new_uuid = get_uuid_from_xml(xml) duplicate_instances = Instance.objects.filter(uuid=new_uuid) if duplicate_instances: # ensure we have saved the extra attachments for f in media_files: Attachment.objects.get_or_create( instance=duplicate_instances[0], media_file=f, mimetype=f.content_type) else: instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) return instance if duplicate_instances: # We are now outside the atomic block, so we can raise an exception # without rolling back the extra attachments we created earlier # NB: Since `ATOMIC_REQUESTS` is set at the database level, everything # could still be rolled back if the calling view fails to handle an # exception raise DuplicateInstance()
def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self)
def create_instance(username, xml_file, media_files, status='submitted_via_web', uuid=None, date_created_override=None, request=None): """ Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() xml_hash = Instance.get_hash(xml) xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) # get new and deprecated uuid's new_uuid = get_uuid_from_xml(xml) # Dorey's rule from 2012 (commit 890a67aa): # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. # The start-time requirement protected submissions with identical responses # from being rejected as duplicates *before* KoBoCAT had the concept of # submission UUIDs. Nowadays, OpenRosa requires clients to send a UUID (in # `<instanceID>`) within every submission; if the incoming XML has a UUID # and still exactly matches an existing submission, it's certainly a # duplicate (https://docs.opendatakit.org/openrosa-metadata/#fields). if xform.has_start_time or new_uuid is not None: # XML matches are identified by identical content hash OR, when a # content hash is not present, by string comparison of the full # content, which is slow! Use the management command # `populate_xml_hashes_for_instances` to hash existing submissions existing_instance = Instance.objects.filter( Q(xml_hash=xml_hash) | Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml), xform__user=xform.user, ).first() else: existing_instance = None if existing_instance: # ensure we have saved the extra attachments any_new_attachment = save_attachments(existing_instance, media_files) if not any_new_attachment: raise DuplicateInstance() else: # Update Mongo via the related ParsedInstance existing_instance.parsed_instance.save(asynchronous=False) return existing_instance else: instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) return instance
def create_new_submission(self, request, site, form): fs_xf = FieldSightXF.objects.get(pk=form) xform = fs_xf.xf xml_file_list = self.request.FILES.pop('xml_submission_file', []) xml_file = xml_file_list[0] if len(xml_file_list) else None xml = xml_file.read() username = self.kwargs.get('username') user = get_object_or_404(User, username=username) media_files = request.FILES.values() new_uuid = get_uuid_from_xml(xml) site_id = site xml_hash = Instance.get_hash(xml) if xform.has_start_time: # XML matches are identified by identical content hash OR, when a # content hash is not present, by string comparison of the full # content, which is slow! Use the management command # `populate_xml_hashes_for_instances` to hash existing submissions existing_instance = Instance.objects.filter( Q(xml_hash=xml_hash) | Q(xml_hash=Instance.DEFAULT_XML_HASH, xml=xml), xform__user=xform.user, ).first() else: existing_instance = None if existing_instance: # ensure we have saved the extra attachments any_new_attachment = save_attachments(existing_instance, media_files) if not any_new_attachment: raise DuplicateInstance() else: context = self.get_serializer_context() serializer = SubmissionSerializer(existing_instance, context=context) return Response(serializer.data, headers=self.get_openrosa_headers(request), status=status.HTTP_201_CREATED, template_name=self.template_name) with transaction.atomic(): if fs_xf.is_survey: instance = save_submission( xform=xform, xml=xml, media_files=media_files, new_uuid=new_uuid, submitted_by=user, status='submitted_via_web', date_created_override=None, fxid=None, site=None, fs_poj_id=fs_xf.id, project=fs_xf.project.id, ) else: if fs_xf.site: instance = save_submission( xform=xform, xml=xml, media_files=media_files, new_uuid=new_uuid, submitted_by=user, status='submitted_via_web', date_created_override=None, fxid=fs_xf.id, site=site_id, ) else: instance = save_submission( xform=xform, xml=xml, media_files=media_files, new_uuid=new_uuid, submitted_by=user, status='submitted_via_web', date_created_override=None, fxid=None, site=site_id, fs_poj_id=fs_xf.id, project=fs_xf.project.id, ) task_obj = CeleryTaskProgress.objects.create( user=user, description='Change site info', task_type=25, content_object=instance.fieldsight_instance) if task_obj: from onadata.apps.fieldsight.tasks import \ update_meta_details update_meta_details.apply_async( (fs_xf.id, instance.id, task_obj.id, site_id), countdown=1) else: from onadata.apps.fieldsight.tasks import \ update_meta_details update_meta_details.apply_async( (fs_xf.id, instance.id, 0, site_id), countdown=1) noti_type = 16 title = "new submission" if instance.fieldsight_instance.site: extra_object = instance.fieldsight_instance.site extra_message = "" project = extra_object.project site = extra_object organization = extra_object.project.organization else: extra_object = instance.fieldsight_instance.project extra_message = "project" project = extra_object site = None organization = extra_object.organization instance.fieldsight_instance.logs.create( source=user, type=noti_type, title=title, organization=organization, project=project, site=site, extra_object=extra_object, extra_message=extra_message, content_object=instance.fieldsight_instance) context = self.get_serializer_context() serializer = SubmissionSerializer(instance, context=context) return Response(serializer.data, headers=self.get_openrosa_headers(request), status=status.HTTP_201_CREATED, template_name=self.template_name)
def create_instance(username, xml_file, media_files, status=u'submitted_via_web', uuid=None, date_created_override=None, request=None): """ I used to check if this file had been submitted already, I've taken this out because it was too slow. Now we're going to create a way for an admin to mark duplicate instances. This should simplify things a bit. Submission cases: If there is a username and no uuid, submitting an old ODK form. If there is a username and a uuid, submitting a new ODK form. """ with transaction.atomic(): instance = None submitted_by = request.user \ if request and request.user.is_authenticated() else None if username: username = username.lower() xml = xml_file.read() xform = get_xform_from_submission(xml, username, uuid) check_submission_permissions(request, xform) existing_instance_count = Instance.objects.filter( xml=xml, xform__user=xform.user).count() if existing_instance_count > 0: existing_instance = Instance.objects.filter( xml=xml, xform__user=xform.user)[0] if not existing_instance.xform or\ existing_instance.xform.has_start_time: # Ignore submission as a duplicate IFF # * a submission's XForm collects start time # * the submitted XML is an exact match with one that # has already been submitted for that user. raise DuplicateInstance() # get new and depracated uuid's new_uuid = get_uuid_from_xml(xml) duplicate_instances = Instance.objects.filter(uuid=new_uuid) if duplicate_instances: # ensure we have saved the extra attachments for f in media_files: Attachment.objects.get_or_create( instance=duplicate_instances[0], media_file=f, mimetype=f.content_type) else: instance = save_submission(xform, xml, media_files, new_uuid, submitted_by, status, date_created_override) return instance if duplicate_instances: # We are now outside the atomic block, so we can raise an exception # without rolling back the extra attachments we created earlier # NB: Since `ATOMIC_REQUESTS` is set at the database level, everything # could still be rolled back if the calling view fails to handle an # exception raise DuplicateInstance()