コード例 #1
0
ファイル: test_parsing.py プロジェクト: onaio/onadata
    def test_parse_xform_nested_repeats(self):
        self._publish_and_submit_new_repeats()
        parser = XFormInstanceParser(self.xml, self.xform)
        dict = parser.to_dict()
        expected_dict = {
            u'new_repeats': {
                u'info':
                {
                    u'age': u'80',
                    u'name': u'Adam'
                },
                u'kids':
                {
                    u'kids_details':
                    [
                        {
                            u'kids_age': u'50',
                            u'kids_name': u'Abel'
                        },
                    ],
                    u'has_kids': u'1'
                },
                u'web_browsers': u'chrome ie',
                u'gps': u'-1.2627557 36.7926442 0.0 30.0'
            }
        }
        self.assertEqual(dict, expected_dict)

        flat_dict = parser.to_flat_dict()
        expected_flat_dict = {
            u'gps': u'-1.2627557 36.7926442 0.0 30.0',
            u'kids/kids_details':
            [
                {
                    u'kids/kids_details/kids_name': u'Abel',
                    u'kids/kids_details/kids_age': u'50'
                }
            ],
            u'kids/has_kids': u'1',
            u'info/age': u'80',
            u'web_browsers': u'chrome ie',
            u'info/name': u'Adam'
        }
        self.assertEqual(flat_dict, expected_flat_dict)
コード例 #2
0
ファイル: test_parsing.py プロジェクト: riboh/onadata
    def test_parse_xform_nested_repeats(self):
        self._publish_and_submit_new_repeats()
        parser = XFormInstanceParser(self.xml, self.xform)
        dict = parser.to_dict()
        expected_dict = {
            u'new_repeats': {
                u'info':
                {
                    u'age': u'80',
                    u'name': u'Adam'
                },
                u'kids':
                {
                    u'kids_details':
                    [
                        {
                            u'kids_age': u'50',
                            u'kids_name': u'Abel'
                        },
                    ],
                    u'has_kids': u'1'
                },
                u'web_browsers': u'chrome ie',
                u'gps': u'-1.2627557 36.7926442 0.0 30.0'
            }
        }
        self.assertEqual(dict, expected_dict)

        flat_dict = parser.to_flat_dict()
        expected_flat_dict = {
            u'gps': u'-1.2627557 36.7926442 0.0 30.0',
            u'kids/kids_details':
            [
                {
                    u'kids/kids_details/kids_name': u'Abel',
                    u'kids/kids_details/kids_age': u'50'
                }
            ],
            u'kids/has_kids': u'1',
            u'info/age': u'80',
            u'web_browsers': u'chrome ie',
            u'info/name': u'Adam'
        }
        self.assertEqual(flat_dict, expected_flat_dict)
コード例 #3
0
ファイル: test_parsing.py プロジェクト: hnjamba/onaclone-2018
    def test_multiple_media_files_on_encrypted_form(self):
        self._create_user_and_login()
        # publish our form which contains some some repeats
        xls_file_path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../fixtures/tutorial_encrypted/tutorial_encrypted.xls")
        count = XForm.objects.count()
        self._publish_xls_file_and_set_xform(xls_file_path)
        self.assertEqual(count + 1, XForm.objects.count())

        # submit an instance
        xml_submission_file_path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml")
        self._make_submission(xml_submission_file_path)
        self.assertNotContains(self.response,
                               "Multiple nodes with the same name",
                               status_code=201)

        # load xml file to parse and compare
        xml_file = open(xml_submission_file_path)
        self.xml = xml_file.read()
        xml_file.close()

        parser = XFormInstanceParser(self.xml, self.xform)
        dict = parser.to_dict()

        expected_list = [{
            u'file': u'1483528430996.jpg.enc'
        }, {
            u'file': u'1483528445767.jpg.enc'
        }]
        self.assertEqual(dict.get('data').get('media'), expected_list)

        flat_dict = parser.to_flat_dict()
        expected_flat_list = [{
            u'media/file': u'1483528430996.jpg.enc'
        }, {
            u'media/file': u'1483528445767.jpg.enc'
        }]
        self.assertEqual(flat_dict.get('media'), expected_flat_list)
コード例 #4
0
ファイル: test_parsing.py プロジェクト: onaio/onadata
    def test_multiple_media_files_on_encrypted_form(self):
        self._create_user_and_login()
        # publish our form which contains some some repeats
        xls_file_path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../fixtures/tutorial_encrypted/tutorial_encrypted.xls"
        )
        count = XForm.objects.count()
        self._publish_xls_file_and_set_xform(xls_file_path)
        self.assertEqual(count + 1, XForm.objects.count())

        # submit an instance
        xml_submission_file_path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml"
        )
        self._make_submission(xml_submission_file_path)
        self.assertNotContains(self.response,
                               "Multiple nodes with the same name",
                               status_code=201)

        # load xml file to parse and compare
        xml_file = open(xml_submission_file_path)
        self.xml = xml_file.read()
        xml_file.close()

        parser = XFormInstanceParser(self.xml, self.xform)
        dict = parser.to_dict()

        expected_list = [{u'file': u'1483528430996.jpg.enc'},
                         {u'file': u'1483528445767.jpg.enc'}]
        self.assertEqual(dict.get('data').get('media'), expected_list)

        flat_dict = parser.to_flat_dict()
        expected_flat_list = [{u'media/file': u'1483528430996.jpg.enc'},
                              {u'media/file': u'1483528445767.jpg.enc'}]
        self.assertEqual(flat_dict.get('media'), expected_flat_list)
コード例 #5
0
ファイル: test_parsing.py プロジェクト: hkmshb/onadata
    def test_parse_xform_nested_repeats(self):
        self._publish_and_submit_new_repeats()
        parser = XFormInstanceParser(self.xml, self.xform.data_dictionary())
        dict = parser.to_dict()
        expected_dict = {
            u"new_repeats": {
                u"info": {u"age": u"80", u"name": u"Adam"},
                u"kids": {u"kids_details": [{u"kids_age": u"50", u"kids_name": u"Abel"}], u"has_kids": u"1"},
                u"web_browsers": u"chrome ie",
                u"gps": u"-1.2627557 36.7926442 0.0 30.0",
            }
        }
        self.assertEqual(dict, expected_dict)

        flat_dict = parser.to_flat_dict()
        expected_flat_dict = {
            u"gps": u"-1.2627557 36.7926442 0.0 30.0",
            u"kids/kids_details": [{u"kids/kids_details/kids_name": u"Abel", u"kids/kids_details/kids_age": u"50"}],
            u"kids/has_kids": u"1",
            u"info/age": u"80",
            u"web_browsers": u"chrome ie",
            u"info/name": u"Adam",
        }
        self.assertEqual(flat_dict, expected_flat_dict)
コード例 #6
0
class InstanceBaseClass(object):
    """Interface of functions for Instance and InstanceHistory model"""
    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def numeric_converter(self, json_dict, numeric_fields=None):
        if numeric_fields is None:
            numeric_fields = get_numeric_fields(self.xform)
        for key, value in json_dict.items():
            if isinstance(value, basestring) and key in numeric_fields:
                converted_value = numeric_checker(value)
                if converted_value:
                    json_dict[key] = converted_value
            elif isinstance(value, dict):
                json_dict[key] = self.numeric_converter(value, numeric_fields)
            elif isinstance(value, list):
                for k, v in enumerate(value):
                    if isinstance(v, basestring) and key in numeric_fields:
                        converted_value = numeric_checker(v)
                        if converted_value:
                            json_dict[key] = converted_value
                    elif isinstance(v, dict):
                        value[k] = self.numeric_converter(v, numeric_fields)
        return json_dict

    def _set_geom(self):
        xform = self.xform
        geo_xpaths = xform.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                for gps in get_values_matching_key(doc, xpath):
                    try:
                        geometry = [float(s) for s in gps.split()]
                        lat, lng = geometry[0:2]
                        points.append(Point(lng, lat))
                    except ValueError:
                        return

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        self.json = self.get_full_dict()

    def get_full_dict(self, load_existing=True):
        doc = self.json or {} if load_existing else {}
        # Get latest dict
        doc = self.get_dict()

        if self.id:
            doc.update({
                UUID:
                self.uuid,
                ID:
                self.id,
                BAMBOO_DATASET_ID:
                self.xform.bamboo_dataset,
                ATTACHMENTS:
                _get_attachments_from_instance(self),
                STATUS:
                self.status,
                TAGS:
                list(self.tags.names()),
                NOTES:
                self.get_notes(),
                VERSION:
                self.version,
                DURATION:
                self.get_duration(),
                XFORM_ID_STRING:
                self._parser.get_xform_id_string(),
                GEOLOCATION:
                [self.point.y, self.point.x] if self.point else [None, None],
                SUBMITTED_BY:
                self.user.username if self.user else None
            })

            for osm in self.osm_data.all():
                doc.update(osm.get_tags_with_prefix())

            if isinstance(self.deleted_at, datetime):
                doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

            if not self.date_created:
                self.date_created = submission_time()

            doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)

            edited = False
            if hasattr(self, 'last_edited'):
                edited = self.last_edited is not None

            doc[EDITED] = edited
            edited and doc.update(
                {LAST_EDITED: convert_to_serializable_date(self.last_edited)})

        return doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(self.xml, self.xform)

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
            else self._parser.to_dict()
        return self.numeric_converter(instance_dict)

    def get_notes(self):
        return [{
            "id": note.id,
            "owner": note.created_by.username,
            "note": note.note,
            "instance_field": note.instance_field,
            "created_by": note.created_by.id
        } for note in self.notes.all()]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    def get_duration(self):
        data = self.get_dict()
        start_name = _get_tag_or_element_type_xpath(self.xform, START)
        end_name = _get_tag_or_element_type_xpath(self.xform, END)
        start_time, end_time = data.get(start_name), data.get(end_name)

        return calculate_duration(start_time, end_time)
コード例 #7
0
class Instance(models.Model):
    json = JSONField(default={}, null=False)
    xml = models.TextField()
    user = models.ForeignKey(User, related_name='instances', null=True)
    xform = models.ForeignKey(XForm, null=True, related_name='instances')
    survey_type = models.ForeignKey(SurveyType)

    # shows when we first received this instance
    date_created = models.DateTimeField(auto_now_add=True)

    # this will end up representing "date last parsed"
    date_modified = models.DateTimeField(auto_now=True)

    # this will end up representing "date instance was deleted"
    deleted_at = models.DateTimeField(null=True, default=None)

    # ODK keeps track of three statuses for an instance:
    # incomplete, submitted, complete
    # we add a fourth status: submitted_via_web
    status = models.CharField(max_length=20, default=u'submitted_via_web')
    uuid = models.CharField(max_length=249, default=u'')

    # store an geographic objects associated with this instance
    geom = models.GeometryCollectionField(null=True)
    objects = models.GeoManager()

    tags = TaggableManager()

    class Meta:
        app_label = 'logger'

    @classmethod
    def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
        try:
            instance = cls.objects.get(id=instance_id)
        except cls.DoesNotExist:
            pass
        else:
            instance.set_deleted(deleted_at)

    def _check_active(self, force):
        """Check that form is active and raise exception if not.

        :param force: Ignore restrictions on saving.
        """
        if not force and self.xform and not self.xform.downloadable:
            raise FormInactiveError()

    def _set_geom(self):
        xform = self.xform
        data_dictionary = xform.data_dictionary()
        geo_xpaths = data_dictionary.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                geometry = [float(s) for s in doc.get(xpath, u'').split()]

                if len(geometry):
                    lat, lng = geometry[0:2]
                    points.append(Point(lng, lat))

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        doc = self.get_dict()

        if not self.date_created:
            now = submission_time()
            self.date_created = now

        point = self.point
        if point:
            doc[GEOLOCATION] = [point.y, point.x]

        doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
        doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
        doc[SUBMITTED_BY] = self.user.username\
            if self.user is not None else None
        self.json = doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(self.xml,
                                               self.xform.data_dictionary())

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        return self._parser.get_flat_dict_with_attributes() if flat else\
            self._parser.to_dict()

    def get_full_dict(self):
        # TODO should we store all of these in the JSON no matter what?
        d = self.json
        data = {
            UUID: self.uuid,
            ID: self.id,
            BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
            self.USERFORM_ID:
            u'%s_%s' % (self.user.username, self.xform.id_string),
            ATTACHMENTS: [a.media_file.name for a in self.attachments.all()],
            self.STATUS: self.status,
            TAGS: list(self.tags.names()),
            NOTES: self.get_notes()
        }

        if isinstance(self.instance.deleted_at, datetime):
            data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

        d.update(data)

        return d

    def get_notes(self):
        return [note['note'] for note in self.notes.values('note')]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def save(self, *args, **kwargs):
        force = kwargs.get('force')

        if force:
            del kwargs['force']

        self._check_active(force)

        self._set_geom()
        self._set_json()
        self._set_survey_type()
        self._set_uuid()
        super(Instance, self).save(*args, **kwargs)

    def set_deleted(self, deleted_at=timezone.now()):
        self.deleted_at = deleted_at
        self.save()
        # force submission count re-calculation
        self.xform.submission_count(force_update=True)
        self.parsed_instance.save()
コード例 #8
0
ファイル: instance.py プロジェクト: boney-bun/kobocat
class Instance(models.Model):
    XML_HASH_LENGTH = 64
    DEFAULT_XML_HASH = None

    json = JSONField(default={}, null=False)
    xml = models.TextField()
    xml_hash = models.CharField(max_length=XML_HASH_LENGTH,
                                db_index=True,
                                null=True,
                                default=DEFAULT_XML_HASH)
    user = models.ForeignKey(User, related_name='instances', null=True)
    xform = models.ForeignKey(XForm, null=True, related_name='instances')
    survey_type = models.ForeignKey(SurveyType)

    # shows when we first received this instance
    date_created = models.DateTimeField(auto_now_add=True)

    # this will end up representing "date last parsed"
    date_modified = models.DateTimeField(auto_now=True)

    # this will end up representing "date instance was deleted"
    deleted_at = models.DateTimeField(null=True, default=None)

    # ODK keeps track of three statuses for an instance:
    # incomplete, submitted, complete
    # we add a fourth status: submitted_via_web
    status = models.CharField(max_length=20, default=u'submitted_via_web')
    uuid = models.CharField(max_length=249, default=u'')

    # store an geographic objects associated with this instance
    geom = models.GeometryCollectionField(null=True)
    objects = models.GeoManager()

    tags = TaggableManager()

    validation_status = JSONField(null=True, default=None)

    class Meta:
        app_label = 'logger'

    @property
    def asset(self):
        """
        The goal of this property is to make the code future proof.
        We can run the tests on kpi backend or kobocat backend.
        Instance.asset will exist for both
        It's used for validation_statuses.
        :return: XForm
        """
        return self.xform

    @classmethod
    def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
        try:
            instance = cls.objects.get(id=instance_id)
        except cls.DoesNotExist:
            pass
        else:
            instance.set_deleted(deleted_at)

    def _check_active(self, force):
        """Check that form is active and raise exception if not.

        :param force: Ignore restrictions on saving.
        """
        if not force and self.xform and not self.xform.downloadable:
            raise FormInactiveError()

    def _set_geom(self):
        xform = self.xform
        data_dictionary = xform.data_dictionary()
        geo_xpaths = data_dictionary.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                geometry = [float(s) for s in doc.get(xpath, u'').split()]

                if len(geometry):
                    lat, lng = geometry[0:2]
                    points.append(Point(lng, lat))

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        doc = self.get_dict()

        if not self.date_created:
            now = submission_time()
            self.date_created = now

        point = self.point
        if point:
            doc[GEOLOCATION] = [point.y, point.x]

        doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
        doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
        doc[SUBMITTED_BY] = self.user.username\
            if self.user is not None else None
        self.json = doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(self.xml,
                                               self.xform.data_dictionary())

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def _populate_xml_hash(self):
        '''
        Populate the `xml_hash` attribute of this `Instance` based on the content of the `xml`
        attribute.
        '''
        self.xml_hash = self.get_hash(self.xml)

    @classmethod
    def populate_xml_hashes_for_instances(cls,
                                          usernames=None,
                                          pk__in=None,
                                          repopulate=False):
        '''
        Populate the `xml_hash` field for `Instance` instances limited to the specified users
        and/or DB primary keys.

        :param list[str] usernames: Optional list of usernames for whom `Instance`s will be
        populated with hashes.
        :param list[int] pk__in: Optional list of primary keys for `Instance`s that should be
        populated with hashes.
        :param bool repopulate: Optional argument to force repopulation of existing hashes.
        :returns: Total number of `Instance`s updated.
        :rtype: int
        '''

        filter_kwargs = dict()
        if usernames:
            filter_kwargs['xform__user__username__in'] = usernames
        if pk__in:
            filter_kwargs['pk__in'] = pk__in
        # By default, skip over instances previously populated with hashes.
        if not repopulate:
            filter_kwargs['xml_hash'] = cls.DEFAULT_XML_HASH

        # Query for the target `Instance`s.
        target_instances_queryset = cls.objects.filter(**filter_kwargs)

        # Exit quickly if there's nothing to do.
        if not target_instances_queryset.exists():
            return 0

        # Limit our queryset result content since we'll only need the `pk` and `xml` attributes.
        target_instances_queryset = target_instances_queryset.only('pk', 'xml')
        instances_updated_total = 0

        # Break the potentially large `target_instances_queryset` into chunks to avoid memory
        # exhaustion.
        chunk_size = 2000
        target_instances_queryset = target_instances_queryset.order_by('pk')
        target_instances_qs_chunk = target_instances_queryset
        while target_instances_qs_chunk.exists():
            # Take a chunk of the target `Instance`s.
            target_instances_qs_chunk = target_instances_qs_chunk[0:chunk_size]

            for instance in target_instances_qs_chunk:
                pk = instance.pk
                xml = instance.xml
                # Do a `Queryset.update()` on this individual instance to avoid signals triggering
                # things like `Reversion` versioning.
                instances_updated_count = Instance.objects.filter(
                    pk=pk).update(xml_hash=cls.get_hash(xml))
                instances_updated_total += instances_updated_count

            # Set up the next chunk
            target_instances_qs_chunk = target_instances_queryset.filter(
                pk__gt=instance.pk)

        return instances_updated_total

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        return self._parser.get_flat_dict_with_attributes() if flat else\
            self._parser.to_dict()

    def get_full_dict(self):
        # TODO should we store all of these in the JSON no matter what?
        d = self.json
        data = {
            UUID: self.uuid,
            ID: self.id,
            BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
            self.USERFORM_ID:
            u'%s_%s' % (self.user.username, self.xform.id_string),
            ATTACHMENTS: [a.media_file.name for a in self.attachments.all()],
            self.STATUS: self.status,
            TAGS: list(self.tags.names()),
            NOTES: self.get_notes()
        }

        if isinstance(self.instance.deleted_at, datetime):
            data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

        d.update(data)

        return d

    def get_notes(self):
        return [note['note'] for note in self.notes.values('note')]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    @staticmethod
    def get_hash(input_string):
        '''
        Compute the SHA256 hash of the given string. A wrapper to standardize hash computation.

        :param basestring input_sting: The string to be hashed.
        :return: The resulting hash.
        :rtype: str
        '''
        if isinstance(input_string, unicode):
            input_string = input_string.encode('utf-8')
        return sha256(input_string).hexdigest()

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def save(self, *args, **kwargs):
        force = kwargs.get('force')

        if force:
            del kwargs['force']

        self._check_active(force)

        self._set_geom()
        self._set_json()
        self._set_survey_type()
        self._set_uuid()
        self._populate_xml_hash()

        # Force validation_status to be dict
        if self.validation_status is None:
            self.validation_status = {}

        super(Instance, self).save(*args, **kwargs)

    def set_deleted(self, deleted_at=timezone.now()):
        self.deleted_at = deleted_at
        self.save()
        # force submission count re-calculation
        self.xform.submission_count(force_update=True)
        self.parsed_instance.save()

    def get_validation_status(self):
        """
        Returns instance validation status.

        :return: object
        """
        # This method can be tweaked to implement default validation status
        # For example:
        # if not self.validation_status:
        #    self.validation_status = self.asset.settings.get("validation_statuses")[0]
        return self.validation_status
コード例 #9
0
ファイル: instance.py プロジェクト: Azique/kobocat
class Instance(models.Model):
    json = JSONField(default={}, null=False)
    xml = models.TextField()
    user = models.ForeignKey(User, related_name='instances', null=True)
    xform = models.ForeignKey(XForm, null=True, related_name='instances')
    survey_type = models.ForeignKey(SurveyType)

    # shows when we first received this instance
    date_created = models.DateTimeField(auto_now_add=True)

    # this will end up representing "date last parsed"
    date_modified = models.DateTimeField(auto_now=True)

    # this will end up representing "date instance was deleted"
    deleted_at = models.DateTimeField(null=True, default=None)

    # ODK keeps track of three statuses for an instance:
    # incomplete, submitted, complete
    # we add a fourth status: submitted_via_web
    status = models.CharField(max_length=20,
                              default=u'submitted_via_web')
    uuid = models.CharField(max_length=249, default=u'')

    # store an geographic objects associated with this instance
    geom = models.GeometryCollectionField(null=True)
    objects = models.GeoManager()

    tags = TaggableManager()

    class Meta:
        app_label = 'logger'

    @classmethod
    def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
        try:
            instance = cls.objects.get(id=instance_id)
        except cls.DoesNotExist:
            pass
        else:
            instance.set_deleted(deleted_at)

    def _check_active(self, force):
        """Check that form is active and raise exception if not.

        :param force: Ignore restrictions on saving.
        """
        if not force and self.xform and not self.xform.downloadable:
            raise FormInactiveError()

    def _set_geom(self):
        xform = self.xform
        data_dictionary = xform.data_dictionary()
        geo_xpaths = data_dictionary.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                geometry = [float(s) for s in doc.get(xpath, u'').split()]

                if len(geometry):
                    lat, lng = geometry[0:2]
                    points.append(Point(lng, lat))

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        doc = self.get_dict()

        if not self.date_created:
            now = submission_time()
            self.date_created = now

        point = self.point
        if point:
            doc[GEOLOCATION] = [point.y, point.x]

        doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
        doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
        doc[SUBMITTED_BY] = self.user.username\
            if self.user is not None else None
        self.json = doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(
                self.xml, self.xform.data_dictionary())

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        return self._parser.get_flat_dict_with_attributes() if flat else\
            self._parser.to_dict()

    def get_full_dict(self):
        # TODO should we store all of these in the JSON no matter what?
        d = self.json
        data = {
            UUID: self.uuid,
            ID: self.id,
            BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
            self.USERFORM_ID: u'%s_%s' % (
                self.user.username,
                self.xform.id_string),
            ATTACHMENTS: [a.media_file.name for a in
                          self.attachments.all()],
            self.STATUS: self.status,
            TAGS: list(self.tags.names()),
            NOTES: self.get_notes()
        }

        if isinstance(self.instance.deleted_at, datetime):
            data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

        d.update(data)

        return d

    def get_notes(self):
        return [note['note'] for note in self.notes.values('note')]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def save(self, *args, **kwargs):
        force = kwargs.get('force')

        if force:
            del kwargs['force']

        self._check_active(force)

        self._set_geom()
        self._set_json()
        self._set_survey_type()
        self._set_uuid()
        super(Instance, self).save(*args, **kwargs)

    def set_deleted(self, deleted_at=timezone.now()):
        self.deleted_at = deleted_at
        self.save()
        # force submission count re-calculation
        self.xform.submission_count(force_update=True)
        self.parsed_instance.save()
コード例 #10
0
ファイル: instance.py プロジェクト: onaio/onadata
class InstanceBaseClass(object):
    """Interface of functions for Instance and InstanceHistory model"""

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def numeric_converter(self, json_dict, numeric_fields=None):
        if numeric_fields is None:
            # pylint: disable=no-member
            numeric_fields = get_numeric_fields(self.xform)
        for key, value in json_dict.items():
            if isinstance(value, basestring) and key in numeric_fields:
                converted_value = numeric_checker(value)
                if converted_value:
                    json_dict[key] = converted_value
            elif isinstance(value, dict):
                json_dict[key] = self.numeric_converter(
                    value, numeric_fields)
            elif isinstance(value, list):
                for k, v in enumerate(value):
                    if isinstance(v, basestring) and key in numeric_fields:
                        converted_value = numeric_checker(v)
                        if converted_value:
                            json_dict[key] = converted_value
                    elif isinstance(v, dict):
                        value[k] = self.numeric_converter(
                            v, numeric_fields)
        return json_dict

    def _set_geom(self):
        # pylint: disable=no-member
        xform = self.xform
        geo_xpaths = xform.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if geo_xpaths:
            for xpath in geo_xpaths:
                for gps in get_values_matching_key(doc, xpath):
                    try:
                        geometry = [float(s) for s in gps.split()]
                        lat, lng = geometry[0:2]
                        points.append(Point(lng, lat))
                    except ValueError:
                        return

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        self.json = self.get_full_dict()

    def get_full_dict(self, load_existing=True):
        doc = self.json or {} if load_existing else {}
        # Get latest dict
        doc = self.get_dict()
        # pylint: disable=no-member
        if self.id:
            doc.update({
                UUID: self.uuid,
                ID: self.id,
                BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
                ATTACHMENTS: _get_attachments_from_instance(self),
                STATUS: self.status,
                TAGS: list(self.tags.names()),
                NOTES: self.get_notes(),
                VERSION: self.version,
                DURATION: self.get_duration(),
                XFORM_ID_STRING: self._parser.get_xform_id_string(),
                XFORM_ID: self.xform.pk,
                GEOLOCATION: [self.point.y, self.point.x] if self.point
                else [None, None],
                SUBMITTED_BY: self.user.username if self.user else None
            })

            for osm in self.osm_data.all():
                doc.update(osm.get_tags_with_prefix())

            if isinstance(self.deleted_at, datetime):
                doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

            # pylint: disable=no-member
            if self.has_a_review:
                status, comment = self.get_review_status_and_comment()
                doc[REVIEW_STATUS] = status
                if comment:
                    doc[REVIEW_COMMENT] = comment

            # pylint: disable=attribute-defined-outside-init
            if not self.date_created:
                self.date_created = submission_time()

            doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)

            doc[TOTAL_MEDIA] = self.total_media
            doc[MEDIA_COUNT] = self.media_count
            doc[MEDIA_ALL_RECEIVED] = self.media_all_received

            edited = False
            if hasattr(self, 'last_edited'):
                edited = self.last_edited is not None

            doc[EDITED] = edited
            edited and doc.update({
                LAST_EDITED: convert_to_serializable_date(self.last_edited)
            })
        return doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            # pylint: disable=no-member
            self._parser = XFormInstanceParser(self.xml, self.xform)

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        # pylint: disable=no-member, attribute-defined-outside-init
        if self.xml and not self.uuid:
            # pylint: disable=no-member
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
            else self._parser.to_dict()
        return self.numeric_converter(instance_dict)

    def get_notes(self):
        # pylint: disable=no-member
        return [note.get_data() for note in self.notes.all()]

    def get_review_status_and_comment(self):
        """
        Return a tuple of review status and comment
        """
        try:
            # pylint: disable=no-member
            status = self.reviews.latest('date_modified').status
            comment = self.reviews.latest('date_modified').get_note_text()
            return status, comment
        except SubmissionReview.DoesNotExist:
            return None

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    def get_duration(self):
        data = self.get_dict()
        # pylint: disable=no-member
        start_name = _get_tag_or_element_type_xpath(self.xform, START)
        end_name = _get_tag_or_element_type_xpath(self.xform, END)
        start_time, end_time = data.get(start_name), data.get(end_name)

        return calculate_duration(start_time, end_time)
コード例 #11
0
ファイル: instance.py プロジェクト: kobotoolbox/kobocat
class Instance(models.Model):
    XML_HASH_LENGTH = 64
    DEFAULT_XML_HASH = None

    json = JSONField(default={}, null=False)
    xml = models.TextField()
    xml_hash = models.CharField(max_length=XML_HASH_LENGTH, db_index=True, null=True,
                                default=DEFAULT_XML_HASH)
    user = models.ForeignKey(User, related_name='instances', null=True)
    xform = models.ForeignKey(XForm, null=True, related_name='instances')
    survey_type = models.ForeignKey(SurveyType)

    # shows when we first received this instance
    date_created = models.DateTimeField(auto_now_add=True)

    # this will end up representing "date last parsed"
    date_modified = models.DateTimeField(auto_now=True)

    # this will end up representing "date instance was deleted"
    deleted_at = models.DateTimeField(null=True, default=None)

    # ODK keeps track of three statuses for an instance:
    # incomplete, submitted, complete
    # we add a fourth status: submitted_via_web
    status = models.CharField(max_length=20,
                              default=u'submitted_via_web')
    uuid = models.CharField(max_length=249, default=u'', db_index=True)

    # store an geographic objects associated with this instance
    geom = models.GeometryCollectionField(null=True)
    objects = models.GeoManager()

    tags = TaggableManager()

    validation_status = JSONField(null=True, default=None)

    # TODO Don't forget to update all records with command `update_is_sync_with_mongo`.
    is_synced_with_mongo = LazyDefaultBooleanField(default=False)

    # If XForm.has_kpi_hooks` is True, this field should be True either.
    # It tells whether the instance has been successfully sent to KPI.
    posted_to_kpi = LazyDefaultBooleanField(default=False)

    class Meta:
        app_label = 'logger'

    @property
    def asset(self):
        """
        The goal of this property is to make the code future proof.
        We can run the tests on kpi backend or kobocat backend.
        Instance.asset will exist for both
        It's used for validation_statuses.
        :return: XForm
        """
        return self.xform

    @classmethod
    def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
        try:
            instance = cls.objects.get(id=instance_id)
        except cls.DoesNotExist:
            pass
        else:
            instance.set_deleted(deleted_at)

    def _check_active(self, force):
        """Check that form is active and raise exception if not.

        :param force: Ignore restrictions on saving.
        """
        if not force and self.xform and not self.xform.downloadable:
            raise FormInactiveError()

    def _set_geom(self):
        xform = self.xform
        data_dictionary = xform.data_dictionary()
        geo_xpaths = data_dictionary.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                geometry = [float(s) for s in doc.get(xpath, u'').split()]

                if len(geometry):
                    lat, lng = geometry[0:2]
                    points.append(Point(lng, lat))

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        doc = self.get_dict()

        if not self.date_created:
            now = submission_time()
            self.date_created = now

        point = self.point
        if point:
            doc[GEOLOCATION] = [point.y, point.x]

        doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
        doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
        doc[SUBMITTED_BY] = self.user.username\
            if self.user is not None else None
        self.json = doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(
                self.xml, self.xform.data_dictionary())

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def _populate_xml_hash(self):
        '''
        Populate the `xml_hash` attribute of this `Instance` based on the content of the `xml`
        attribute.
        '''
        self.xml_hash = self.get_hash(self.xml)

    @classmethod
    def populate_xml_hashes_for_instances(cls, usernames=None, pk__in=None, repopulate=False):
        '''
        Populate the `xml_hash` field for `Instance` instances limited to the specified users
        and/or DB primary keys.

        :param list[str] usernames: Optional list of usernames for whom `Instance`s will be
        populated with hashes.
        :param list[int] pk__in: Optional list of primary keys for `Instance`s that should be
        populated with hashes.
        :param bool repopulate: Optional argument to force repopulation of existing hashes.
        :returns: Total number of `Instance`s updated.
        :rtype: int
        '''

        filter_kwargs = dict()
        if usernames:
            filter_kwargs['xform__user__username__in'] = usernames
        if pk__in:
            filter_kwargs['pk__in'] = pk__in
        # By default, skip over instances previously populated with hashes.
        if not repopulate:
            filter_kwargs['xml_hash'] = cls.DEFAULT_XML_HASH

        # Query for the target `Instance`s.
        target_instances_queryset = cls.objects.filter(**filter_kwargs)

        # Exit quickly if there's nothing to do.
        if not target_instances_queryset.exists():
            return 0

        # Limit our queryset result content since we'll only need the `pk` and `xml` attributes.
        target_instances_queryset = target_instances_queryset.only('pk', 'xml')
        instances_updated_total = 0

        # Break the potentially large `target_instances_queryset` into chunks to avoid memory
        # exhaustion.
        chunk_size = 2000
        target_instances_queryset = target_instances_queryset.order_by('pk')
        target_instances_qs_chunk = target_instances_queryset
        while target_instances_qs_chunk.exists():
            # Take a chunk of the target `Instance`s.
            target_instances_qs_chunk = target_instances_qs_chunk[0:chunk_size]

            for instance in target_instances_qs_chunk:
                pk = instance.pk
                xml = instance.xml
                # Do a `Queryset.update()` on this individual instance to avoid signals triggering
                # things like `Reversion` versioning.
                instances_updated_count = Instance.objects.filter(pk=pk).update(
                    xml_hash=cls.get_hash(xml))
                instances_updated_total += instances_updated_count

            # Set up the next chunk
            target_instances_qs_chunk = target_instances_queryset.filter(
                pk__gt=instance.pk)

        return instances_updated_total

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        return self._parser.get_flat_dict_with_attributes() if flat else\
            self._parser.to_dict()

    def get_full_dict(self):
        # TODO should we store all of these in the JSON no matter what?
        d = self.json
        data = {
            UUID: self.uuid,
            ID: self.id,
            BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
            self.USERFORM_ID: u'%s_%s' % (
                self.user.username,
                self.xform.id_string),
            ATTACHMENTS: [a.media_file.name for a in
                          self.attachments.all()],
            self.STATUS: self.status,
            TAGS: list(self.tags.names()),
            NOTES: self.get_notes()
        }

        if isinstance(self.instance.deleted_at, datetime):
            data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

        d.update(data)

        return d

    def get_notes(self):
        return [note['note'] for note in self.notes.values('note')]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    @staticmethod
    def get_hash(input_string):
        '''
        Compute the SHA256 hash of the given string. A wrapper to standardize hash computation.

        :param basestring input_sting: The string to be hashed.
        :return: The resulting hash.
        :rtype: str
        '''
        if isinstance(input_string, unicode):
            input_string = input_string.encode('utf-8')
        return sha256(input_string).hexdigest()

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def save(self, *args, **kwargs):
        force = kwargs.pop("force", False)

        self._check_active(force)

        self._set_geom()
        self._set_json()
        self._set_survey_type()
        self._set_uuid()
        self._populate_xml_hash()

        # Force validation_status to be dict
        if self.validation_status is None:
            self.validation_status = {}

        super(Instance, self).save(*args, **kwargs)

    def set_deleted(self, deleted_at=timezone.now()):
        self.deleted_at = deleted_at
        self.save()
        # force submission count re-calculation
        self.xform.submission_count(force_update=True)
        self.parsed_instance.save()

    def get_validation_status(self):
        """
        Returns instance validation status.

        :return: object
        """
        # This method can be tweaked to implement default validation status
        # For example:
        # if not self.validation_status:
        #    self.validation_status = self.asset.settings.get("validation_statuses")[0]
        return self.validation_status
コード例 #12
0
ファイル: instance.py プロジェクト: babacar/onadata
class InstanceBaseClass(object):
    """Interface of functions for Instance and InstanceHistory model"""

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def numeric_converter(self, json_dict, numeric_fields=None):
        if numeric_fields is None:
            # pylint: disable=no-member
            numeric_fields = get_numeric_fields(self.xform)
        for key, value in json_dict.items():
            if isinstance(value, basestring) and key in numeric_fields:
                converted_value = numeric_checker(value)
                if converted_value:
                    json_dict[key] = converted_value
            elif isinstance(value, dict):
                json_dict[key] = self.numeric_converter(
                    value, numeric_fields)
            elif isinstance(value, list):
                for k, v in enumerate(value):
                    if isinstance(v, basestring) and key in numeric_fields:
                        converted_value = numeric_checker(v)
                        if converted_value:
                            json_dict[key] = converted_value
                    elif isinstance(v, dict):
                        value[k] = self.numeric_converter(
                            v, numeric_fields)
        return json_dict

    def _set_geom(self):
        # pylint: disable=no-member
        xform = self.xform
        geo_xpaths = xform.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if geo_xpaths:
            for xpath in geo_xpaths:
                for gps in get_values_matching_key(doc, xpath):
                    try:
                        geometry = [float(s) for s in gps.split()]
                        lat, lng = geometry[0:2]
                        points.append(Point(lng, lat))
                    except ValueError:
                        return

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        self.json = self.get_full_dict()

    def get_full_dict(self, load_existing=True):
        doc = self.json or {} if load_existing else {}
        # Get latest dict
        doc = self.get_dict()
        # pylint: disable=no-member
        if self.id:
            doc.update({
                UUID: self.uuid,
                ID: self.id,
                BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
                ATTACHMENTS: _get_attachments_from_instance(self),
                STATUS: self.status,
                TAGS: list(self.tags.names()),
                NOTES: self.get_notes(),
                VERSION: self.version,
                DURATION: self.get_duration(),
                XFORM_ID_STRING: self._parser.get_xform_id_string(),
                XFORM_ID: self.xform.pk,
                GEOLOCATION: [self.point.y, self.point.x] if self.point
                else [None, None],
                SUBMITTED_BY: self.user.username if self.user else None
            })

            for osm in self.osm_data.all():
                doc.update(osm.get_tags_with_prefix())

            if isinstance(self.deleted_at, datetime):
                doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

            # pylint: disable=no-member
            if self.has_a_review:
                review = self.get_latest_review()
                if review:
                    doc[REVIEW_STATUS] = review.status
                    doc[REVIEW_DATE] = review.date_created.strftime(
                        MONGO_STRFTIME)
                    if review.get_note_text():
                        doc[REVIEW_COMMENT] = review.get_note_text()

            # pylint: disable=attribute-defined-outside-init
            if not self.date_created:
                self.date_created = submission_time()

            if not self.date_modified:
                self.date_modified = self.date_created

            doc[DATE_MODIFIED] = self.date_modified.strftime(
                    MONGO_STRFTIME)

            doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)

            doc[TOTAL_MEDIA] = self.total_media
            doc[MEDIA_COUNT] = self.media_count
            doc[MEDIA_ALL_RECEIVED] = self.media_all_received

            edited = False
            if hasattr(self, 'last_edited'):
                edited = self.last_edited is not None

            doc[EDITED] = edited
            edited and doc.update({
                LAST_EDITED: convert_to_serializable_date(self.last_edited)
            })
        return doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            # pylint: disable=no-member
            self._parser = XFormInstanceParser(self.xml, self.xform)

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        # pylint: disable=no-member, attribute-defined-outside-init
        if self.xml and not self.uuid:
            # pylint: disable=no-member
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
            else self._parser.to_dict()
        return self.numeric_converter(instance_dict)

    def get_notes(self):
        # pylint: disable=no-member
        return [note.get_data() for note in self.notes.all()]

    @deprecated(version='2.5.3',
                reason="Deprecated in favour of `get_latest_review`")
    def get_review_status_and_comment(self):
        """
        Return a tuple of review status and comment.
        """
        try:
            # pylint: disable=no-member
            status = self.reviews.latest('date_modified').status
            comment = self.reviews.latest('date_modified').get_note_text()
            return status, comment
        except SubmissionReview.DoesNotExist:
            return None

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    def get_duration(self):
        data = self.get_dict()
        # pylint: disable=no-member
        start_name = _get_tag_or_element_type_xpath(self.xform, START)
        end_name = _get_tag_or_element_type_xpath(self.xform, END)
        start_time, end_time = data.get(start_name), data.get(end_name)

        return calculate_duration(start_time, end_time)

    def get_latest_review(self):
        """
        Returns the latest review.
        Used in favour of `get_review_status_and_comment`.
        """
        try:
            return self.reviews.latest('date_modified')
        except SubmissionReview.DoesNotExist:
            return None
コード例 #13
0
class Instance(models.Model):
    json = JSONField(default={}, null=False)
    xml = models.TextField()
    user = models.ForeignKey(User, related_name='instances', null=True)
    xform = models.ForeignKey(XForm, null=True, related_name='instances')
    survey_type = models.ForeignKey(SurveyType)

    # shows when we first received this instance
    date_created = models.DateTimeField(auto_now_add=True)

    # this will end up representing "date last parsed"
    date_modified = models.DateTimeField(auto_now=True)

    # this will end up representing "date instance was deleted"
    deleted_at = models.DateTimeField(null=True, default=None)

    # ODK keeps track of three statuses for an instance:
    # incomplete, submitted, complete
    # we add a fourth status: submitted_via_web
    status = models.CharField(max_length=20, default=u'submitted_via_web')
    uuid = models.CharField(max_length=249, default=u'')
    version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True)

    # store an geographic objects associated with this instance
    geom = models.GeometryCollectionField(null=True)
    objects = models.GeoManager()

    tags = TaggableManager()

    class Meta:
        app_label = 'logger'

    @classmethod
    def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
        try:
            instance = cls.objects.get(id=instance_id)
        except cls.DoesNotExist:
            pass
        else:
            instance.set_deleted(deleted_at)

    def numeric_converter(self, json_dict, numeric_fields=None):
        if numeric_fields is None:
            numeric_fields = get_numeric_fields(self.xform)
        for key, value in json_dict.items():
            if isinstance(value, basestring) and key in numeric_fields:
                converted_value = numeric_checker(value)
                if converted_value:
                    json_dict[key] = converted_value
            elif isinstance(value, dict):
                json_dict[key] = self.numeric_converter(value, numeric_fields)
            elif isinstance(value, list):
                for k, v in enumerate(value):
                    if isinstance(v, basestring) and key in numeric_fields:
                        converted_value = numeric_checker(v)
                        if converted_value:
                            json_dict[key] = converted_value
                    elif isinstance(v, dict):
                        value[k] = self.numeric_converter(v, numeric_fields)
        return json_dict

    def _check_active(self, force):
        """Check that form is active and raise exception if not.

        :param force: Ignore restrictions on saving.
        """
        if not force and self.xform and not self.xform.downloadable:
            raise FormInactiveError()

    def _set_geom(self):
        xform = self.xform
        data_dictionary = xform.data_dictionary()
        geo_xpaths = data_dictionary.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                geometry = [float(s) for s in doc.get(xpath, u'').split()]

                if len(geometry):
                    lat, lng = geometry[0:2]
                    points.append(Point(lng, lat))

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        self.json = self.get_full_dict()

    def get_full_dict(self):
        doc = self.json or {}
        doc.update(self.get_dict())

        if self.id:
            doc.update({
                UUID:
                self.uuid,
                ID:
                self.id,
                BAMBOO_DATASET_ID:
                self.xform.bamboo_dataset,
                ATTACHMENTS:
                _get_attachments_from_instance(self),
                STATUS:
                self.status,
                TAGS:
                list(self.tags.names()),
                NOTES:
                self.get_notes(),
                VERSION:
                self.version,
                DURATION:
                self.get_duration(),
                XFORM_ID_STRING:
                self._parser.get_xform_id_string(),
                GEOLOCATION:
                [self.point.y, self.point.x] if self.point else [None, None],
                SUBMITTED_BY:
                self.user.username if self.user else None
            })

            if isinstance(self.deleted_at, datetime):
                doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

            if not self.date_created:
                self.date_created = submission_time()

            doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)

        return doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(self.xml,
                                               self.xform.data_dictionary())

    def _set_survey_type(self):
        self.survey_type, created = \
            SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
            else self._parser.to_dict()
        return self.numeric_converter(instance_dict)

    def get_notes(self):
        return [note['note'] for note in self.notes.values('note')]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def save(self, *args, **kwargs):
        force = kwargs.get('force')

        if force:
            del kwargs['force']

        self._check_active(force)

        self._set_geom()
        self._set_json()
        self._set_survey_type()
        self._set_uuid()
        self.version = self.xform.version
        super(Instance, self).save(*args, **kwargs)

    def set_deleted(self, deleted_at=timezone.now()):
        self.deleted_at = deleted_at
        self.save()
        # force submission count re-calculation
        self.xform.submission_count(force_update=True)
        self.parsed_instance.save()

    def get_duration(self):
        data = self.get_dict()
        dd = self.xform.data_dictionary()
        start_name = _get_tag_or_element_type_xpath(dd, START)
        end_name = _get_tag_or_element_type_xpath(dd, END)
        start_time, end_time = data.get(start_name), data.get(end_name)

        return calculate_duration(start_time, end_time)
コード例 #14
0
ファイル: instance.py プロジェクト: hkmshb/onadata
class Instance(models.Model):
    json = JSONField(default={}, null=False)
    xml = models.TextField()
    user = models.ForeignKey(User, related_name="instances", null=True)
    xform = models.ForeignKey(XForm, null=True, related_name="instances")
    survey_type = models.ForeignKey(SurveyType)

    # shows when we first received this instance
    date_created = models.DateTimeField(auto_now_add=True)

    # this will end up representing "date last parsed"
    date_modified = models.DateTimeField(auto_now=True)

    # this will end up representing "date instance was deleted"
    deleted_at = models.DateTimeField(null=True, default=None)

    # ODK keeps track of three statuses for an instance:
    # incomplete, submitted, complete
    # we add a fourth status: submitted_via_web
    status = models.CharField(max_length=20, default=u"submitted_via_web")
    uuid = models.CharField(max_length=249, default=u"")
    version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True)

    # store an geographic objects associated with this instance
    geom = models.GeometryCollectionField(null=True)
    objects = models.GeoManager()

    tags = TaggableManager()

    class Meta:
        app_label = "logger"

    @classmethod
    def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
        try:
            instance = cls.objects.get(id=instance_id)
        except cls.DoesNotExist:
            pass
        else:
            instance.set_deleted(deleted_at)

    def numeric_converter(self, json_dict, numeric_fields=None):
        if numeric_fields is None:
            numeric_fields = get_numeric_fields(self.xform)
        for key, value in json_dict.items():
            if isinstance(value, basestring) and key in numeric_fields:
                converted_value = numeric_checker(value)
                if converted_value:
                    json_dict[key] = converted_value
            elif isinstance(value, dict):
                json_dict[key] = self.numeric_converter(value, numeric_fields)
            elif isinstance(value, list):
                for k, v in enumerate(value):
                    if isinstance(v, basestring) and key in numeric_fields:
                        converted_value = numeric_checker(v)
                        if converted_value:
                            json_dict[key] = converted_value
                    elif isinstance(v, dict):
                        value[k] = self.numeric_converter(v, numeric_fields)
        return json_dict

    def _check_active(self, force):
        """Check that form is active and raise exception if not.

        :param force: Ignore restrictions on saving.
        """
        if not force and self.xform and not self.xform.downloadable:
            raise FormInactiveError()

    def _set_geom(self):
        xform = self.xform
        data_dictionary = xform.data_dictionary()
        geo_xpaths = data_dictionary.geopoint_xpaths()
        doc = self.get_dict()
        points = []

        if len(geo_xpaths):
            for xpath in geo_xpaths:
                geometry = [float(s) for s in doc.get(xpath, u"").split()]

                if len(geometry):
                    lat, lng = geometry[0:2]
                    points.append(Point(lng, lat))

            if not xform.instances_with_geopoints and len(points):
                xform.instances_with_geopoints = True
                xform.save()

            self.geom = GeometryCollection(points)

    def _set_json(self):
        self.json = self.get_full_dict()

    def get_full_dict(self):
        doc = self.json or {}
        doc.update(self.get_dict())

        if self.id:
            doc.update(
                {
                    UUID: self.uuid,
                    ID: self.id,
                    BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
                    ATTACHMENTS: _get_attachments_from_instance(self),
                    STATUS: self.status,
                    TAGS: list(self.tags.names()),
                    NOTES: self.get_notes(),
                    VERSION: self.version,
                    DURATION: self.get_duration(),
                    XFORM_ID_STRING: self._parser.get_xform_id_string(),
                    GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None],
                    SUBMITTED_BY: self.user.username if self.user else None,
                }
            )

            if isinstance(self.deleted_at, datetime):
                doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)

            if not self.date_created:
                self.date_created = submission_time()

            doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)

        return doc

    def _set_parser(self):
        if not hasattr(self, "_parser"):
            self._parser = XFormInstanceParser(self.xml, self.xform.data_dictionary())

    def _set_survey_type(self):
        self.survey_type, created = SurveyType.objects.get_or_create(slug=self.get_root_node_name())

    def _set_uuid(self):
        if self.xml and not self.uuid:
            uuid = get_uuid_from_xml(self.xml)
            if uuid is not None:
                self.uuid = uuid
        set_uuid(self)

    def get(self, abbreviated_xpath):
        self._set_parser()
        return self._parser.get(abbreviated_xpath)

    def get_dict(self, force_new=False, flat=True):
        """Return a python object representation of this instance's XML."""
        self._set_parser()

        instance_dict = self._parser.get_flat_dict_with_attributes() if flat else self._parser.to_dict()
        return self.numeric_converter(instance_dict)

    def get_notes(self):
        return [note["note"] for note in self.notes.values("note")]

    def get_root_node(self):
        self._set_parser()
        return self._parser.get_root_node()

    def get_root_node_name(self):
        self._set_parser()
        return self._parser.get_root_node_name()

    @property
    def point(self):
        gc = self.geom

        if gc and len(gc):
            return gc[0]

    def save(self, *args, **kwargs):
        force = kwargs.get("force")

        if force:
            del kwargs["force"]

        self._check_active(force)

        self._set_geom()
        self._set_json()
        self._set_survey_type()
        self._set_uuid()
        self.version = self.xform.version
        super(Instance, self).save(*args, **kwargs)

    def set_deleted(self, deleted_at=timezone.now()):
        self.deleted_at = deleted_at
        self.save()
        # force submission count re-calculation
        self.xform.submission_count(force_update=True)
        self.parsed_instance.save()

    def get_duration(self):
        data = self.get_dict()
        dd = self.xform.data_dictionary()
        start_name = _get_tag_or_element_type_xpath(dd, START)
        end_name = _get_tag_or_element_type_xpath(dd, END)
        start_time, end_time = data.get(start_name), data.get(end_name)

        return calculate_duration(start_time, end_time)