def test_parse_xform_nested_repeats(self): self._publish_and_submit_new_repeats() parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() expected_dict = { u'new_repeats': { u'info': { u'age': u'80', u'name': u'Adam' }, u'kids': { u'kids_details': [ { u'kids_age': u'50', u'kids_name': u'Abel' }, ], u'has_kids': u'1' }, u'web_browsers': u'chrome ie', u'gps': u'-1.2627557 36.7926442 0.0 30.0' } } self.assertEqual(dict, expected_dict) flat_dict = parser.to_flat_dict() expected_flat_dict = { u'gps': u'-1.2627557 36.7926442 0.0 30.0', u'kids/kids_details': [ { u'kids/kids_details/kids_name': u'Abel', u'kids/kids_details/kids_age': u'50' } ], u'kids/has_kids': u'1', u'info/age': u'80', u'web_browsers': u'chrome ie', u'info/name': u'Adam' } self.assertEqual(flat_dict, expected_flat_dict)
def test_multiple_media_files_on_encrypted_form(self): self._create_user_and_login() # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial_encrypted/tutorial_encrypted.xls") count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) self.assertEqual(count + 1, XForm.objects.count()) # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml") self._make_submission(xml_submission_file_path) self.assertNotContains(self.response, "Multiple nodes with the same name", status_code=201) # load xml file to parse and compare xml_file = open(xml_submission_file_path) self.xml = xml_file.read() xml_file.close() parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() expected_list = [{ u'file': u'1483528430996.jpg.enc' }, { u'file': u'1483528445767.jpg.enc' }] self.assertEqual(dict.get('data').get('media'), expected_list) flat_dict = parser.to_flat_dict() expected_flat_list = [{ u'media/file': u'1483528430996.jpg.enc' }, { u'media/file': u'1483528445767.jpg.enc' }] self.assertEqual(flat_dict.get('media'), expected_flat_list)
def test_multiple_media_files_on_encrypted_form(self): self._create_user_and_login() # publish our form which contains some some repeats xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial_encrypted/tutorial_encrypted.xls" ) count = XForm.objects.count() self._publish_xls_file_and_set_xform(xls_file_path) self.assertEqual(count + 1, XForm.objects.count()) # submit an instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml" ) self._make_submission(xml_submission_file_path) self.assertNotContains(self.response, "Multiple nodes with the same name", status_code=201) # load xml file to parse and compare xml_file = open(xml_submission_file_path) self.xml = xml_file.read() xml_file.close() parser = XFormInstanceParser(self.xml, self.xform) dict = parser.to_dict() expected_list = [{u'file': u'1483528430996.jpg.enc'}, {u'file': u'1483528445767.jpg.enc'}] self.assertEqual(dict.get('data').get('media'), expected_list) flat_dict = parser.to_flat_dict() expected_flat_list = [{u'media/file': u'1483528430996.jpg.enc'}, {u'media/file': u'1483528445767.jpg.enc'}] self.assertEqual(flat_dict.get('media'), expected_flat_list)
def test_parse_xform_nested_repeats(self): self._publish_and_submit_new_repeats() parser = XFormInstanceParser(self.xml, self.xform.data_dictionary()) dict = parser.to_dict() expected_dict = { u"new_repeats": { u"info": {u"age": u"80", u"name": u"Adam"}, u"kids": {u"kids_details": [{u"kids_age": u"50", u"kids_name": u"Abel"}], u"has_kids": u"1"}, u"web_browsers": u"chrome ie", u"gps": u"-1.2627557 36.7926442 0.0 30.0", } } self.assertEqual(dict, expected_dict) flat_dict = parser.to_flat_dict() expected_flat_dict = { u"gps": u"-1.2627557 36.7926442 0.0 30.0", u"kids/kids_details": [{u"kids/kids_details/kids_name": u"Abel", u"kids/kids_details/kids_age": u"50"}], u"kids/has_kids": u"1", u"info/age": u"80", u"web_browsers": u"chrome ie", u"info/name": u"Adam", } self.assertEqual(flat_dict, expected_flat_dict)
class InstanceBaseClass(object): """Interface of functions for Instance and InstanceHistory model""" @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def numeric_converter(self, json_dict, numeric_fields=None): if numeric_fields is None: numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): if isinstance(value, basestring) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): json_dict[key] = self.numeric_converter(value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): if isinstance(v, basestring) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): value[k] = self.numeric_converter(v, numeric_fields) return json_dict def _set_geom(self): xform = self.xform geo_xpaths = xform.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: for gps in get_values_matching_key(doc, xpath): try: geometry = [float(s) for s in gps.split()] lat, lng = geometry[0:2] points.append(Point(lng, lat)) except ValueError: return if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): self.json = self.get_full_dict() def get_full_dict(self, load_existing=True): doc = self.json or {} if load_existing else {} # Get latest dict doc = self.get_dict() if self.id: doc.update({ UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, ATTACHMENTS: _get_attachments_from_instance(self), STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes(), VERSION: self.version, DURATION: self.get_duration(), XFORM_ID_STRING: self._parser.get_xform_id_string(), GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None], SUBMITTED_BY: self.user.username if self.user else None }) for osm in self.osm_data.all(): doc.update(osm.get_tags_with_prefix()) if isinstance(self.deleted_at, datetime): doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) if not self.date_created: self.date_created = submission_time() doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) edited = False if hasattr(self, 'last_edited'): edited = self.last_edited is not None doc[EDITED] = edited edited and doc.update( {LAST_EDITED: convert_to_serializable_date(self.last_edited)}) return doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ else self._parser.to_dict() return self.numeric_converter(instance_dict) def get_notes(self): return [{ "id": note.id, "owner": note.created_by.username, "note": note.note, "instance_field": note.instance_field, "created_by": note.created_by.id } for note in self.notes.all()] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() def get_duration(self): data = self.get_dict() start_name = _get_tag_or_element_type_xpath(self.xform, START) end_name = _get_tag_or_element_type_xpath(self.xform, END) start_time, end_time = data.get(start_name), data.get(end_name) return calculate_duration(start_time, end_time)
class Instance(models.Model): json = JSONField(default={}, null=False) xml = models.TextField() user = models.ForeignKey(User, related_name='instances', null=True) xform = models.ForeignKey(XForm, null=True, related_name='instances') survey_type = models.ForeignKey(SurveyType) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u'submitted_via_web') uuid = models.CharField(max_length=249, default=u'') # store an geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) objects = models.GeoManager() tags = TaggableManager() class Meta: app_label = 'logger' @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: pass else: instance.set_deleted(deleted_at) def _check_active(self, force): """Check that form is active and raise exception if not. :param force: Ignore restrictions on saving. """ if not force and self.xform and not self.xform.downloadable: raise FormInactiveError() def _set_geom(self): xform = self.xform data_dictionary = xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: geometry = [float(s) for s in doc.get(xpath, u'').split()] if len(geometry): lat, lng = geometry[0:2] points.append(Point(lng, lat)) if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): doc = self.get_dict() if not self.date_created: now = submission_time() self.date_created = now point = self.point if point: doc[GEOLOCATION] = [point.y, point.x] doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() doc[SUBMITTED_BY] = self.user.username\ if self.user is not None else None self.json = doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser(self.xml, self.xform.data_dictionary()) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() return self._parser.get_flat_dict_with_attributes() if flat else\ self._parser.to_dict() def get_full_dict(self): # TODO should we store all of these in the JSON no matter what? d = self.json data = { UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, self.USERFORM_ID: u'%s_%s' % (self.user.username, self.xform.id_string), ATTACHMENTS: [a.media_file.name for a in self.attachments.all()], self.STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes() } if isinstance(self.instance.deleted_at, datetime): data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) d.update(data) return d def get_notes(self): return [note['note'] for note in self.notes.values('note')] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def save(self, *args, **kwargs): force = kwargs.get('force') if force: del kwargs['force'] self._check_active(force) self._set_geom() self._set_json() self._set_survey_type() self._set_uuid() super(Instance, self).save(*args, **kwargs) def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() # force submission count re-calculation self.xform.submission_count(force_update=True) self.parsed_instance.save()
class Instance(models.Model): XML_HASH_LENGTH = 64 DEFAULT_XML_HASH = None json = JSONField(default={}, null=False) xml = models.TextField() xml_hash = models.CharField(max_length=XML_HASH_LENGTH, db_index=True, null=True, default=DEFAULT_XML_HASH) user = models.ForeignKey(User, related_name='instances', null=True) xform = models.ForeignKey(XForm, null=True, related_name='instances') survey_type = models.ForeignKey(SurveyType) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u'submitted_via_web') uuid = models.CharField(max_length=249, default=u'') # store an geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) objects = models.GeoManager() tags = TaggableManager() validation_status = JSONField(null=True, default=None) class Meta: app_label = 'logger' @property def asset(self): """ The goal of this property is to make the code future proof. We can run the tests on kpi backend or kobocat backend. Instance.asset will exist for both It's used for validation_statuses. :return: XForm """ return self.xform @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: pass else: instance.set_deleted(deleted_at) def _check_active(self, force): """Check that form is active and raise exception if not. :param force: Ignore restrictions on saving. """ if not force and self.xform and not self.xform.downloadable: raise FormInactiveError() def _set_geom(self): xform = self.xform data_dictionary = xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: geometry = [float(s) for s in doc.get(xpath, u'').split()] if len(geometry): lat, lng = geometry[0:2] points.append(Point(lng, lat)) if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): doc = self.get_dict() if not self.date_created: now = submission_time() self.date_created = now point = self.point if point: doc[GEOLOCATION] = [point.y, point.x] doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() doc[SUBMITTED_BY] = self.user.username\ if self.user is not None else None self.json = doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser(self.xml, self.xform.data_dictionary()) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def _populate_xml_hash(self): ''' Populate the `xml_hash` attribute of this `Instance` based on the content of the `xml` attribute. ''' self.xml_hash = self.get_hash(self.xml) @classmethod def populate_xml_hashes_for_instances(cls, usernames=None, pk__in=None, repopulate=False): ''' Populate the `xml_hash` field for `Instance` instances limited to the specified users and/or DB primary keys. :param list[str] usernames: Optional list of usernames for whom `Instance`s will be populated with hashes. :param list[int] pk__in: Optional list of primary keys for `Instance`s that should be populated with hashes. :param bool repopulate: Optional argument to force repopulation of existing hashes. :returns: Total number of `Instance`s updated. :rtype: int ''' filter_kwargs = dict() if usernames: filter_kwargs['xform__user__username__in'] = usernames if pk__in: filter_kwargs['pk__in'] = pk__in # By default, skip over instances previously populated with hashes. if not repopulate: filter_kwargs['xml_hash'] = cls.DEFAULT_XML_HASH # Query for the target `Instance`s. target_instances_queryset = cls.objects.filter(**filter_kwargs) # Exit quickly if there's nothing to do. if not target_instances_queryset.exists(): return 0 # Limit our queryset result content since we'll only need the `pk` and `xml` attributes. target_instances_queryset = target_instances_queryset.only('pk', 'xml') instances_updated_total = 0 # Break the potentially large `target_instances_queryset` into chunks to avoid memory # exhaustion. chunk_size = 2000 target_instances_queryset = target_instances_queryset.order_by('pk') target_instances_qs_chunk = target_instances_queryset while target_instances_qs_chunk.exists(): # Take a chunk of the target `Instance`s. target_instances_qs_chunk = target_instances_qs_chunk[0:chunk_size] for instance in target_instances_qs_chunk: pk = instance.pk xml = instance.xml # Do a `Queryset.update()` on this individual instance to avoid signals triggering # things like `Reversion` versioning. instances_updated_count = Instance.objects.filter( pk=pk).update(xml_hash=cls.get_hash(xml)) instances_updated_total += instances_updated_count # Set up the next chunk target_instances_qs_chunk = target_instances_queryset.filter( pk__gt=instance.pk) return instances_updated_total def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() return self._parser.get_flat_dict_with_attributes() if flat else\ self._parser.to_dict() def get_full_dict(self): # TODO should we store all of these in the JSON no matter what? d = self.json data = { UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, self.USERFORM_ID: u'%s_%s' % (self.user.username, self.xform.id_string), ATTACHMENTS: [a.media_file.name for a in self.attachments.all()], self.STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes() } if isinstance(self.instance.deleted_at, datetime): data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) d.update(data) return d def get_notes(self): return [note['note'] for note in self.notes.values('note')] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() @staticmethod def get_hash(input_string): ''' Compute the SHA256 hash of the given string. A wrapper to standardize hash computation. :param basestring input_sting: The string to be hashed. :return: The resulting hash. :rtype: str ''' if isinstance(input_string, unicode): input_string = input_string.encode('utf-8') return sha256(input_string).hexdigest() @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def save(self, *args, **kwargs): force = kwargs.get('force') if force: del kwargs['force'] self._check_active(force) self._set_geom() self._set_json() self._set_survey_type() self._set_uuid() self._populate_xml_hash() # Force validation_status to be dict if self.validation_status is None: self.validation_status = {} super(Instance, self).save(*args, **kwargs) def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() # force submission count re-calculation self.xform.submission_count(force_update=True) self.parsed_instance.save() def get_validation_status(self): """ Returns instance validation status. :return: object """ # This method can be tweaked to implement default validation status # For example: # if not self.validation_status: # self.validation_status = self.asset.settings.get("validation_statuses")[0] return self.validation_status
class Instance(models.Model): json = JSONField(default={}, null=False) xml = models.TextField() user = models.ForeignKey(User, related_name='instances', null=True) xform = models.ForeignKey(XForm, null=True, related_name='instances') survey_type = models.ForeignKey(SurveyType) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u'submitted_via_web') uuid = models.CharField(max_length=249, default=u'') # store an geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) objects = models.GeoManager() tags = TaggableManager() class Meta: app_label = 'logger' @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: pass else: instance.set_deleted(deleted_at) def _check_active(self, force): """Check that form is active and raise exception if not. :param force: Ignore restrictions on saving. """ if not force and self.xform and not self.xform.downloadable: raise FormInactiveError() def _set_geom(self): xform = self.xform data_dictionary = xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: geometry = [float(s) for s in doc.get(xpath, u'').split()] if len(geometry): lat, lng = geometry[0:2] points.append(Point(lng, lat)) if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): doc = self.get_dict() if not self.date_created: now = submission_time() self.date_created = now point = self.point if point: doc[GEOLOCATION] = [point.y, point.x] doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() doc[SUBMITTED_BY] = self.user.username\ if self.user is not None else None self.json = doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser( self.xml, self.xform.data_dictionary()) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() return self._parser.get_flat_dict_with_attributes() if flat else\ self._parser.to_dict() def get_full_dict(self): # TODO should we store all of these in the JSON no matter what? d = self.json data = { UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, self.USERFORM_ID: u'%s_%s' % ( self.user.username, self.xform.id_string), ATTACHMENTS: [a.media_file.name for a in self.attachments.all()], self.STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes() } if isinstance(self.instance.deleted_at, datetime): data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) d.update(data) return d def get_notes(self): return [note['note'] for note in self.notes.values('note')] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def save(self, *args, **kwargs): force = kwargs.get('force') if force: del kwargs['force'] self._check_active(force) self._set_geom() self._set_json() self._set_survey_type() self._set_uuid() super(Instance, self).save(*args, **kwargs) def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() # force submission count re-calculation self.xform.submission_count(force_update=True) self.parsed_instance.save()
class InstanceBaseClass(object): """Interface of functions for Instance and InstanceHistory model""" @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def numeric_converter(self, json_dict, numeric_fields=None): if numeric_fields is None: # pylint: disable=no-member numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): if isinstance(value, basestring) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): json_dict[key] = self.numeric_converter( value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): if isinstance(v, basestring) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): value[k] = self.numeric_converter( v, numeric_fields) return json_dict def _set_geom(self): # pylint: disable=no-member xform = self.xform geo_xpaths = xform.geopoint_xpaths() doc = self.get_dict() points = [] if geo_xpaths: for xpath in geo_xpaths: for gps in get_values_matching_key(doc, xpath): try: geometry = [float(s) for s in gps.split()] lat, lng = geometry[0:2] points.append(Point(lng, lat)) except ValueError: return if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): self.json = self.get_full_dict() def get_full_dict(self, load_existing=True): doc = self.json or {} if load_existing else {} # Get latest dict doc = self.get_dict() # pylint: disable=no-member if self.id: doc.update({ UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, ATTACHMENTS: _get_attachments_from_instance(self), STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes(), VERSION: self.version, DURATION: self.get_duration(), XFORM_ID_STRING: self._parser.get_xform_id_string(), XFORM_ID: self.xform.pk, GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None], SUBMITTED_BY: self.user.username if self.user else None }) for osm in self.osm_data.all(): doc.update(osm.get_tags_with_prefix()) if isinstance(self.deleted_at, datetime): doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) # pylint: disable=no-member if self.has_a_review: status, comment = self.get_review_status_and_comment() doc[REVIEW_STATUS] = status if comment: doc[REVIEW_COMMENT] = comment # pylint: disable=attribute-defined-outside-init if not self.date_created: self.date_created = submission_time() doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[TOTAL_MEDIA] = self.total_media doc[MEDIA_COUNT] = self.media_count doc[MEDIA_ALL_RECEIVED] = self.media_all_received edited = False if hasattr(self, 'last_edited'): edited = self.last_edited is not None doc[EDITED] = edited edited and doc.update({ LAST_EDITED: convert_to_serializable_date(self.last_edited) }) return doc def _set_parser(self): if not hasattr(self, "_parser"): # pylint: disable=no-member self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): # pylint: disable=no-member, attribute-defined-outside-init if self.xml and not self.uuid: # pylint: disable=no-member uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ else self._parser.to_dict() return self.numeric_converter(instance_dict) def get_notes(self): # pylint: disable=no-member return [note.get_data() for note in self.notes.all()] def get_review_status_and_comment(self): """ Return a tuple of review status and comment """ try: # pylint: disable=no-member status = self.reviews.latest('date_modified').status comment = self.reviews.latest('date_modified').get_note_text() return status, comment except SubmissionReview.DoesNotExist: return None def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() def get_duration(self): data = self.get_dict() # pylint: disable=no-member start_name = _get_tag_or_element_type_xpath(self.xform, START) end_name = _get_tag_or_element_type_xpath(self.xform, END) start_time, end_time = data.get(start_name), data.get(end_name) return calculate_duration(start_time, end_time)
class Instance(models.Model): XML_HASH_LENGTH = 64 DEFAULT_XML_HASH = None json = JSONField(default={}, null=False) xml = models.TextField() xml_hash = models.CharField(max_length=XML_HASH_LENGTH, db_index=True, null=True, default=DEFAULT_XML_HASH) user = models.ForeignKey(User, related_name='instances', null=True) xform = models.ForeignKey(XForm, null=True, related_name='instances') survey_type = models.ForeignKey(SurveyType) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u'submitted_via_web') uuid = models.CharField(max_length=249, default=u'', db_index=True) # store an geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) objects = models.GeoManager() tags = TaggableManager() validation_status = JSONField(null=True, default=None) # TODO Don't forget to update all records with command `update_is_sync_with_mongo`. is_synced_with_mongo = LazyDefaultBooleanField(default=False) # If XForm.has_kpi_hooks` is True, this field should be True either. # It tells whether the instance has been successfully sent to KPI. posted_to_kpi = LazyDefaultBooleanField(default=False) class Meta: app_label = 'logger' @property def asset(self): """ The goal of this property is to make the code future proof. We can run the tests on kpi backend or kobocat backend. Instance.asset will exist for both It's used for validation_statuses. :return: XForm """ return self.xform @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: pass else: instance.set_deleted(deleted_at) def _check_active(self, force): """Check that form is active and raise exception if not. :param force: Ignore restrictions on saving. """ if not force and self.xform and not self.xform.downloadable: raise FormInactiveError() def _set_geom(self): xform = self.xform data_dictionary = xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: geometry = [float(s) for s in doc.get(xpath, u'').split()] if len(geometry): lat, lng = geometry[0:2] points.append(Point(lng, lat)) if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): doc = self.get_dict() if not self.date_created: now = submission_time() self.date_created = now point = self.point if point: doc[GEOLOCATION] = [point.y, point.x] doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[XFORM_ID_STRING] = self._parser.get_xform_id_string() doc[SUBMITTED_BY] = self.user.username\ if self.user is not None else None self.json = doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser( self.xml, self.xform.data_dictionary()) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def _populate_xml_hash(self): ''' Populate the `xml_hash` attribute of this `Instance` based on the content of the `xml` attribute. ''' self.xml_hash = self.get_hash(self.xml) @classmethod def populate_xml_hashes_for_instances(cls, usernames=None, pk__in=None, repopulate=False): ''' Populate the `xml_hash` field for `Instance` instances limited to the specified users and/or DB primary keys. :param list[str] usernames: Optional list of usernames for whom `Instance`s will be populated with hashes. :param list[int] pk__in: Optional list of primary keys for `Instance`s that should be populated with hashes. :param bool repopulate: Optional argument to force repopulation of existing hashes. :returns: Total number of `Instance`s updated. :rtype: int ''' filter_kwargs = dict() if usernames: filter_kwargs['xform__user__username__in'] = usernames if pk__in: filter_kwargs['pk__in'] = pk__in # By default, skip over instances previously populated with hashes. if not repopulate: filter_kwargs['xml_hash'] = cls.DEFAULT_XML_HASH # Query for the target `Instance`s. target_instances_queryset = cls.objects.filter(**filter_kwargs) # Exit quickly if there's nothing to do. if not target_instances_queryset.exists(): return 0 # Limit our queryset result content since we'll only need the `pk` and `xml` attributes. target_instances_queryset = target_instances_queryset.only('pk', 'xml') instances_updated_total = 0 # Break the potentially large `target_instances_queryset` into chunks to avoid memory # exhaustion. chunk_size = 2000 target_instances_queryset = target_instances_queryset.order_by('pk') target_instances_qs_chunk = target_instances_queryset while target_instances_qs_chunk.exists(): # Take a chunk of the target `Instance`s. target_instances_qs_chunk = target_instances_qs_chunk[0:chunk_size] for instance in target_instances_qs_chunk: pk = instance.pk xml = instance.xml # Do a `Queryset.update()` on this individual instance to avoid signals triggering # things like `Reversion` versioning. instances_updated_count = Instance.objects.filter(pk=pk).update( xml_hash=cls.get_hash(xml)) instances_updated_total += instances_updated_count # Set up the next chunk target_instances_qs_chunk = target_instances_queryset.filter( pk__gt=instance.pk) return instances_updated_total def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() return self._parser.get_flat_dict_with_attributes() if flat else\ self._parser.to_dict() def get_full_dict(self): # TODO should we store all of these in the JSON no matter what? d = self.json data = { UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, self.USERFORM_ID: u'%s_%s' % ( self.user.username, self.xform.id_string), ATTACHMENTS: [a.media_file.name for a in self.attachments.all()], self.STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes() } if isinstance(self.instance.deleted_at, datetime): data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) d.update(data) return d def get_notes(self): return [note['note'] for note in self.notes.values('note')] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() @staticmethod def get_hash(input_string): ''' Compute the SHA256 hash of the given string. A wrapper to standardize hash computation. :param basestring input_sting: The string to be hashed. :return: The resulting hash. :rtype: str ''' if isinstance(input_string, unicode): input_string = input_string.encode('utf-8') return sha256(input_string).hexdigest() @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def save(self, *args, **kwargs): force = kwargs.pop("force", False) self._check_active(force) self._set_geom() self._set_json() self._set_survey_type() self._set_uuid() self._populate_xml_hash() # Force validation_status to be dict if self.validation_status is None: self.validation_status = {} super(Instance, self).save(*args, **kwargs) def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() # force submission count re-calculation self.xform.submission_count(force_update=True) self.parsed_instance.save() def get_validation_status(self): """ Returns instance validation status. :return: object """ # This method can be tweaked to implement default validation status # For example: # if not self.validation_status: # self.validation_status = self.asset.settings.get("validation_statuses")[0] return self.validation_status
class InstanceBaseClass(object): """Interface of functions for Instance and InstanceHistory model""" @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def numeric_converter(self, json_dict, numeric_fields=None): if numeric_fields is None: # pylint: disable=no-member numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): if isinstance(value, basestring) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): json_dict[key] = self.numeric_converter( value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): if isinstance(v, basestring) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): value[k] = self.numeric_converter( v, numeric_fields) return json_dict def _set_geom(self): # pylint: disable=no-member xform = self.xform geo_xpaths = xform.geopoint_xpaths() doc = self.get_dict() points = [] if geo_xpaths: for xpath in geo_xpaths: for gps in get_values_matching_key(doc, xpath): try: geometry = [float(s) for s in gps.split()] lat, lng = geometry[0:2] points.append(Point(lng, lat)) except ValueError: return if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): self.json = self.get_full_dict() def get_full_dict(self, load_existing=True): doc = self.json or {} if load_existing else {} # Get latest dict doc = self.get_dict() # pylint: disable=no-member if self.id: doc.update({ UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, ATTACHMENTS: _get_attachments_from_instance(self), STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes(), VERSION: self.version, DURATION: self.get_duration(), XFORM_ID_STRING: self._parser.get_xform_id_string(), XFORM_ID: self.xform.pk, GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None], SUBMITTED_BY: self.user.username if self.user else None }) for osm in self.osm_data.all(): doc.update(osm.get_tags_with_prefix()) if isinstance(self.deleted_at, datetime): doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) # pylint: disable=no-member if self.has_a_review: review = self.get_latest_review() if review: doc[REVIEW_STATUS] = review.status doc[REVIEW_DATE] = review.date_created.strftime( MONGO_STRFTIME) if review.get_note_text(): doc[REVIEW_COMMENT] = review.get_note_text() # pylint: disable=attribute-defined-outside-init if not self.date_created: self.date_created = submission_time() if not self.date_modified: self.date_modified = self.date_created doc[DATE_MODIFIED] = self.date_modified.strftime( MONGO_STRFTIME) doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) doc[TOTAL_MEDIA] = self.total_media doc[MEDIA_COUNT] = self.media_count doc[MEDIA_ALL_RECEIVED] = self.media_all_received edited = False if hasattr(self, 'last_edited'): edited = self.last_edited is not None doc[EDITED] = edited edited and doc.update({ LAST_EDITED: convert_to_serializable_date(self.last_edited) }) return doc def _set_parser(self): if not hasattr(self, "_parser"): # pylint: disable=no-member self._parser = XFormInstanceParser(self.xml, self.xform) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): # pylint: disable=no-member, attribute-defined-outside-init if self.xml and not self.uuid: # pylint: disable=no-member uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ else self._parser.to_dict() return self.numeric_converter(instance_dict) def get_notes(self): # pylint: disable=no-member return [note.get_data() for note in self.notes.all()] @deprecated(version='2.5.3', reason="Deprecated in favour of `get_latest_review`") def get_review_status_and_comment(self): """ Return a tuple of review status and comment. """ try: # pylint: disable=no-member status = self.reviews.latest('date_modified').status comment = self.reviews.latest('date_modified').get_note_text() return status, comment except SubmissionReview.DoesNotExist: return None def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() def get_duration(self): data = self.get_dict() # pylint: disable=no-member start_name = _get_tag_or_element_type_xpath(self.xform, START) end_name = _get_tag_or_element_type_xpath(self.xform, END) start_time, end_time = data.get(start_name), data.get(end_name) return calculate_duration(start_time, end_time) def get_latest_review(self): """ Returns the latest review. Used in favour of `get_review_status_and_comment`. """ try: return self.reviews.latest('date_modified') except SubmissionReview.DoesNotExist: return None
class Instance(models.Model): json = JSONField(default={}, null=False) xml = models.TextField() user = models.ForeignKey(User, related_name='instances', null=True) xform = models.ForeignKey(XForm, null=True, related_name='instances') survey_type = models.ForeignKey(SurveyType) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u'submitted_via_web') uuid = models.CharField(max_length=249, default=u'') version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True) # store an geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) objects = models.GeoManager() tags = TaggableManager() class Meta: app_label = 'logger' @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: pass else: instance.set_deleted(deleted_at) def numeric_converter(self, json_dict, numeric_fields=None): if numeric_fields is None: numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): if isinstance(value, basestring) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): json_dict[key] = self.numeric_converter(value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): if isinstance(v, basestring) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): value[k] = self.numeric_converter(v, numeric_fields) return json_dict def _check_active(self, force): """Check that form is active and raise exception if not. :param force: Ignore restrictions on saving. """ if not force and self.xform and not self.xform.downloadable: raise FormInactiveError() def _set_geom(self): xform = self.xform data_dictionary = xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: geometry = [float(s) for s in doc.get(xpath, u'').split()] if len(geometry): lat, lng = geometry[0:2] points.append(Point(lng, lat)) if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): self.json = self.get_full_dict() def get_full_dict(self): doc = self.json or {} doc.update(self.get_dict()) if self.id: doc.update({ UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, ATTACHMENTS: _get_attachments_from_instance(self), STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes(), VERSION: self.version, DURATION: self.get_duration(), XFORM_ID_STRING: self._parser.get_xform_id_string(), GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None], SUBMITTED_BY: self.user.username if self.user else None }) if isinstance(self.deleted_at, datetime): doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) if not self.date_created: self.date_created = submission_time() doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) return doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser(self.xml, self.xform.data_dictionary()) def _set_survey_type(self): self.survey_type, created = \ SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() instance_dict = self._parser.get_flat_dict_with_attributes() if flat \ else self._parser.to_dict() return self.numeric_converter(instance_dict) def get_notes(self): return [note['note'] for note in self.notes.values('note')] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def save(self, *args, **kwargs): force = kwargs.get('force') if force: del kwargs['force'] self._check_active(force) self._set_geom() self._set_json() self._set_survey_type() self._set_uuid() self.version = self.xform.version super(Instance, self).save(*args, **kwargs) def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() # force submission count re-calculation self.xform.submission_count(force_update=True) self.parsed_instance.save() def get_duration(self): data = self.get_dict() dd = self.xform.data_dictionary() start_name = _get_tag_or_element_type_xpath(dd, START) end_name = _get_tag_or_element_type_xpath(dd, END) start_time, end_time = data.get(start_name), data.get(end_name) return calculate_duration(start_time, end_time)
class Instance(models.Model): json = JSONField(default={}, null=False) xml = models.TextField() user = models.ForeignKey(User, related_name="instances", null=True) xform = models.ForeignKey(XForm, null=True, related_name="instances") survey_type = models.ForeignKey(SurveyType) # shows when we first received this instance date_created = models.DateTimeField(auto_now_add=True) # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) # this will end up representing "date instance was deleted" deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: # incomplete, submitted, complete # we add a fourth status: submitted_via_web status = models.CharField(max_length=20, default=u"submitted_via_web") uuid = models.CharField(max_length=249, default=u"") version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True) # store an geographic objects associated with this instance geom = models.GeometryCollectionField(null=True) objects = models.GeoManager() tags = TaggableManager() class Meta: app_label = "logger" @classmethod def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): try: instance = cls.objects.get(id=instance_id) except cls.DoesNotExist: pass else: instance.set_deleted(deleted_at) def numeric_converter(self, json_dict, numeric_fields=None): if numeric_fields is None: numeric_fields = get_numeric_fields(self.xform) for key, value in json_dict.items(): if isinstance(value, basestring) and key in numeric_fields: converted_value = numeric_checker(value) if converted_value: json_dict[key] = converted_value elif isinstance(value, dict): json_dict[key] = self.numeric_converter(value, numeric_fields) elif isinstance(value, list): for k, v in enumerate(value): if isinstance(v, basestring) and key in numeric_fields: converted_value = numeric_checker(v) if converted_value: json_dict[key] = converted_value elif isinstance(v, dict): value[k] = self.numeric_converter(v, numeric_fields) return json_dict def _check_active(self, force): """Check that form is active and raise exception if not. :param force: Ignore restrictions on saving. """ if not force and self.xform and not self.xform.downloadable: raise FormInactiveError() def _set_geom(self): xform = self.xform data_dictionary = xform.data_dictionary() geo_xpaths = data_dictionary.geopoint_xpaths() doc = self.get_dict() points = [] if len(geo_xpaths): for xpath in geo_xpaths: geometry = [float(s) for s in doc.get(xpath, u"").split()] if len(geometry): lat, lng = geometry[0:2] points.append(Point(lng, lat)) if not xform.instances_with_geopoints and len(points): xform.instances_with_geopoints = True xform.save() self.geom = GeometryCollection(points) def _set_json(self): self.json = self.get_full_dict() def get_full_dict(self): doc = self.json or {} doc.update(self.get_dict()) if self.id: doc.update( { UUID: self.uuid, ID: self.id, BAMBOO_DATASET_ID: self.xform.bamboo_dataset, ATTACHMENTS: _get_attachments_from_instance(self), STATUS: self.status, TAGS: list(self.tags.names()), NOTES: self.get_notes(), VERSION: self.version, DURATION: self.get_duration(), XFORM_ID_STRING: self._parser.get_xform_id_string(), GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None], SUBMITTED_BY: self.user.username if self.user else None, } ) if isinstance(self.deleted_at, datetime): doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) if not self.date_created: self.date_created = submission_time() doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME) return doc def _set_parser(self): if not hasattr(self, "_parser"): self._parser = XFormInstanceParser(self.xml, self.xform.data_dictionary()) def _set_survey_type(self): self.survey_type, created = SurveyType.objects.get_or_create(slug=self.get_root_node_name()) def _set_uuid(self): if self.xml and not self.uuid: uuid = get_uuid_from_xml(self.xml) if uuid is not None: self.uuid = uuid set_uuid(self) def get(self, abbreviated_xpath): self._set_parser() return self._parser.get(abbreviated_xpath) def get_dict(self, force_new=False, flat=True): """Return a python object representation of this instance's XML.""" self._set_parser() instance_dict = self._parser.get_flat_dict_with_attributes() if flat else self._parser.to_dict() return self.numeric_converter(instance_dict) def get_notes(self): return [note["note"] for note in self.notes.values("note")] def get_root_node(self): self._set_parser() return self._parser.get_root_node() def get_root_node_name(self): self._set_parser() return self._parser.get_root_node_name() @property def point(self): gc = self.geom if gc and len(gc): return gc[0] def save(self, *args, **kwargs): force = kwargs.get("force") if force: del kwargs["force"] self._check_active(force) self._set_geom() self._set_json() self._set_survey_type() self._set_uuid() self.version = self.xform.version super(Instance, self).save(*args, **kwargs) def set_deleted(self, deleted_at=timezone.now()): self.deleted_at = deleted_at self.save() # force submission count re-calculation self.xform.submission_count(force_update=True) self.parsed_instance.save() def get_duration(self): data = self.get_dict() dd = self.xform.data_dictionary() start_name = _get_tag_or_element_type_xpath(dd, START) end_name = _get_tag_or_element_type_xpath(dd, END) start_time, end_time = data.get(start_name), data.get(end_name) return calculate_duration(start_time, end_time)