def test_parse_xform_nested_repeats(self):
self._publish_and_submit_new_repeats()
parser = XFormInstanceParser(self.xml, self.xform)
dict = parser.to_dict()
expected_dict = {
u'new_repeats': {
u'info':
{
u'age': u'80',
u'name': u'Adam'
},
u'kids':
{
u'kids_details':
[
{
u'kids_age': u'50',
u'kids_name': u'Abel'
},
],
u'has_kids': u'1'
},
u'web_browsers': u'chrome ie',
u'gps': u'-1.2627557 36.7926442 0.0 30.0'
}
}
self.assertEqual(dict, expected_dict)
flat_dict = parser.to_flat_dict()
expected_flat_dict = {
u'gps': u'-1.2627557 36.7926442 0.0 30.0',
u'kids/kids_details':
[
{
u'kids/kids_details/kids_name': u'Abel',
u'kids/kids_details/kids_age': u'50'
}
],
u'kids/has_kids': u'1',
u'info/age': u'80',
u'web_browsers': u'chrome ie',
u'info/name': u'Adam'
}
self.assertEqual(flat_dict, expected_flat_dict)
def test_parse_xform_nested_repeats(self):
self._publish_and_submit_new_repeats()
parser = XFormInstanceParser(self.xml, self.xform)
dict = parser.to_dict()
expected_dict = {
u'new_repeats': {
u'info':
{
u'age': u'80',
u'name': u'Adam'
},
u'kids':
{
u'kids_details':
[
{
u'kids_age': u'50',
u'kids_name': u'Abel'
},
],
u'has_kids': u'1'
},
u'web_browsers': u'chrome ie',
u'gps': u'-1.2627557 36.7926442 0.0 30.0'
}
}
self.assertEqual(dict, expected_dict)
flat_dict = parser.to_flat_dict()
expected_flat_dict = {
u'gps': u'-1.2627557 36.7926442 0.0 30.0',
u'kids/kids_details':
[
{
u'kids/kids_details/kids_name': u'Abel',
u'kids/kids_details/kids_age': u'50'
}
],
u'kids/has_kids': u'1',
u'info/age': u'80',
u'web_browsers': u'chrome ie',
u'info/name': u'Adam'
}
self.assertEqual(flat_dict, expected_flat_dict)
def test_multiple_media_files_on_encrypted_form(self):
self._create_user_and_login()
# publish our form which contains some some repeats
xls_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../fixtures/tutorial_encrypted/tutorial_encrypted.xls")
count = XForm.objects.count()
self._publish_xls_file_and_set_xform(xls_file_path)
self.assertEqual(count + 1, XForm.objects.count())
# submit an instance
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml")
self._make_submission(xml_submission_file_path)
self.assertNotContains(self.response,
"Multiple nodes with the same name",
status_code=201)
# load xml file to parse and compare
xml_file = open(xml_submission_file_path)
self.xml = xml_file.read()
xml_file.close()
parser = XFormInstanceParser(self.xml, self.xform)
dict = parser.to_dict()
expected_list = [{
u'file': u'1483528430996.jpg.enc'
}, {
u'file': u'1483528445767.jpg.enc'
}]
self.assertEqual(dict.get('data').get('media'), expected_list)
flat_dict = parser.to_flat_dict()
expected_flat_list = [{
u'media/file': u'1483528430996.jpg.enc'
}, {
u'media/file': u'1483528445767.jpg.enc'
}]
self.assertEqual(flat_dict.get('media'), expected_flat_list)
def test_multiple_media_files_on_encrypted_form(self):
self._create_user_and_login()
# publish our form which contains some some repeats
xls_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../fixtures/tutorial_encrypted/tutorial_encrypted.xls"
)
count = XForm.objects.count()
self._publish_xls_file_and_set_xform(xls_file_path)
self.assertEqual(count + 1, XForm.objects.count())
# submit an instance
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../fixtures/tutorial_encrypted/instances/tutorial_encrypted.xml"
)
self._make_submission(xml_submission_file_path)
self.assertNotContains(self.response,
"Multiple nodes with the same name",
status_code=201)
# load xml file to parse and compare
xml_file = open(xml_submission_file_path)
self.xml = xml_file.read()
xml_file.close()
parser = XFormInstanceParser(self.xml, self.xform)
dict = parser.to_dict()
expected_list = [{u'file': u'1483528430996.jpg.enc'},
{u'file': u'1483528445767.jpg.enc'}]
self.assertEqual(dict.get('data').get('media'), expected_list)
flat_dict = parser.to_flat_dict()
expected_flat_list = [{u'media/file': u'1483528430996.jpg.enc'},
{u'media/file': u'1483528445767.jpg.enc'}]
self.assertEqual(flat_dict.get('media'), expected_flat_list)
def test_parse_xform_nested_repeats(self):
self._publish_and_submit_new_repeats()
parser = XFormInstanceParser(self.xml, self.xform.data_dictionary())
dict = parser.to_dict()
expected_dict = {
u"new_repeats": {
u"info": {u"age": u"80", u"name": u"Adam"},
u"kids": {u"kids_details": [{u"kids_age": u"50", u"kids_name": u"Abel"}], u"has_kids": u"1"},
u"web_browsers": u"chrome ie",
u"gps": u"-1.2627557 36.7926442 0.0 30.0",
}
}
self.assertEqual(dict, expected_dict)
flat_dict = parser.to_flat_dict()
expected_flat_dict = {
u"gps": u"-1.2627557 36.7926442 0.0 30.0",
u"kids/kids_details": [{u"kids/kids_details/kids_name": u"Abel", u"kids/kids_details/kids_age": u"50"}],
u"kids/has_kids": u"1",
u"info/age": u"80",
u"web_browsers": u"chrome ie",
u"info/name": u"Adam",
}
self.assertEqual(flat_dict, expected_flat_dict)
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml,
self.xform_instance.xform)
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml,
self.xform.data_dictionary())
class InstanceBaseClass(object):
"""Interface of functions for Instance and InstanceHistory model"""
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def numeric_converter(self, json_dict, numeric_fields=None):
if numeric_fields is None:
numeric_fields = get_numeric_fields(self.xform)
for key, value in json_dict.items():
if isinstance(value, basestring) and key in numeric_fields:
converted_value = numeric_checker(value)
if converted_value:
json_dict[key] = converted_value
elif isinstance(value, dict):
json_dict[key] = self.numeric_converter(value, numeric_fields)
elif isinstance(value, list):
for k, v in enumerate(value):
if isinstance(v, basestring) and key in numeric_fields:
converted_value = numeric_checker(v)
if converted_value:
json_dict[key] = converted_value
elif isinstance(v, dict):
value[k] = self.numeric_converter(v, numeric_fields)
return json_dict
def _set_geom(self):
xform = self.xform
geo_xpaths = xform.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
for gps in get_values_matching_key(doc, xpath):
try:
geometry = [float(s) for s in gps.split()]
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
except ValueError:
return
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
self.json = self.get_full_dict()
def get_full_dict(self, load_existing=True):
doc = self.json or {} if load_existing else {}
# Get latest dict
doc = self.get_dict()
if self.id:
doc.update({
UUID:
self.uuid,
ID:
self.id,
BAMBOO_DATASET_ID:
self.xform.bamboo_dataset,
ATTACHMENTS:
_get_attachments_from_instance(self),
STATUS:
self.status,
TAGS:
list(self.tags.names()),
NOTES:
self.get_notes(),
VERSION:
self.version,
DURATION:
self.get_duration(),
XFORM_ID_STRING:
self._parser.get_xform_id_string(),
GEOLOCATION:
[self.point.y, self.point.x] if self.point else [None, None],
SUBMITTED_BY:
self.user.username if self.user else None
})
for osm in self.osm_data.all():
doc.update(osm.get_tags_with_prefix())
if isinstance(self.deleted_at, datetime):
doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
if not self.date_created:
self.date_created = submission_time()
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
edited = False
if hasattr(self, 'last_edited'):
edited = self.last_edited is not None
doc[EDITED] = edited
edited and doc.update(
{LAST_EDITED: convert_to_serializable_date(self.last_edited)})
return doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml, self.xform)
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
else self._parser.to_dict()
return self.numeric_converter(instance_dict)
def get_notes(self):
return [{
"id": note.id,
"owner": note.created_by.username,
"note": note.note,
"instance_field": note.instance_field,
"created_by": note.created_by.id
} for note in self.notes.all()]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
def get_duration(self):
data = self.get_dict()
start_name = _get_tag_or_element_type_xpath(self.xform, START)
end_name = _get_tag_or_element_type_xpath(self.xform, END)
start_time, end_time = data.get(start_name), data.get(end_name)
return calculate_duration(start_time, end_time)
def _set_parser(self):
if not hasattr(self, "_parser"):
# pylint: disable=E1101
self._parser = XFormInstanceParser(self.xml, self.xform)
class Instance(models.Model):
json = JSONField(default={}, null=False)
xml = models.TextField()
user = models.ForeignKey(User, related_name='instances', null=True)
xform = models.ForeignKey(XForm, null=True, related_name='instances')
survey_type = models.ForeignKey(SurveyType)
# shows when we first received this instance
date_created = models.DateTimeField(auto_now_add=True)
# this will end up representing "date last parsed"
date_modified = models.DateTimeField(auto_now=True)
# this will end up representing "date instance was deleted"
deleted_at = models.DateTimeField(null=True, default=None)
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
# we add a fourth status: submitted_via_web
status = models.CharField(max_length=20, default=u'submitted_via_web')
uuid = models.CharField(max_length=249, default=u'')
# store an geographic objects associated with this instance
geom = models.GeometryCollectionField(null=True)
objects = models.GeoManager()
tags = TaggableManager()
class Meta:
app_label = 'logger'
@classmethod
def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
try:
instance = cls.objects.get(id=instance_id)
except cls.DoesNotExist:
pass
else:
instance.set_deleted(deleted_at)
def _check_active(self, force):
"""Check that form is active and raise exception if not.
:param force: Ignore restrictions on saving.
"""
if not force and self.xform and not self.xform.downloadable:
raise FormInactiveError()
def _set_geom(self):
xform = self.xform
data_dictionary = xform.data_dictionary()
geo_xpaths = data_dictionary.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
geometry = [float(s) for s in doc.get(xpath, u'').split()]
if len(geometry):
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
doc = self.get_dict()
if not self.date_created:
now = submission_time()
self.date_created = now
point = self.point
if point:
doc[GEOLOCATION] = [point.y, point.x]
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
doc[SUBMITTED_BY] = self.user.username\
if self.user is not None else None
self.json = doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml,
self.xform.data_dictionary())
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
return self._parser.get_flat_dict_with_attributes() if flat else\
self._parser.to_dict()
def get_full_dict(self):
# TODO should we store all of these in the JSON no matter what?
d = self.json
data = {
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
self.USERFORM_ID:
u'%s_%s' % (self.user.username, self.xform.id_string),
ATTACHMENTS: [a.media_file.name for a in self.attachments.all()],
self.STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes()
}
if isinstance(self.instance.deleted_at, datetime):
data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
d.update(data)
return d
def get_notes(self):
return [note['note'] for note in self.notes.values('note')]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def save(self, *args, **kwargs):
force = kwargs.get('force')
if force:
del kwargs['force']
self._check_active(force)
self._set_geom()
self._set_json()
self._set_survey_type()
self._set_uuid()
super(Instance, self).save(*args, **kwargs)
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
# force submission count re-calculation
self.xform.submission_count(force_update=True)
self.parsed_instance.save()
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(
self.xml, self.xform.data_dictionary())
class Instance(models.Model):
XML_HASH_LENGTH = 64
DEFAULT_XML_HASH = None
json = JSONField(default={}, null=False)
xml = models.TextField()
xml_hash = models.CharField(max_length=XML_HASH_LENGTH,
db_index=True,
null=True,
default=DEFAULT_XML_HASH)
user = models.ForeignKey(User, related_name='instances', null=True)
xform = models.ForeignKey(XForm, null=True, related_name='instances')
survey_type = models.ForeignKey(SurveyType)
# shows when we first received this instance
date_created = models.DateTimeField(auto_now_add=True)
# this will end up representing "date last parsed"
date_modified = models.DateTimeField(auto_now=True)
# this will end up representing "date instance was deleted"
deleted_at = models.DateTimeField(null=True, default=None)
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
# we add a fourth status: submitted_via_web
status = models.CharField(max_length=20, default=u'submitted_via_web')
uuid = models.CharField(max_length=249, default=u'')
# store an geographic objects associated with this instance
geom = models.GeometryCollectionField(null=True)
objects = models.GeoManager()
tags = TaggableManager()
validation_status = JSONField(null=True, default=None)
class Meta:
app_label = 'logger'
@property
def asset(self):
"""
The goal of this property is to make the code future proof.
We can run the tests on kpi backend or kobocat backend.
Instance.asset will exist for both
It's used for validation_statuses.
:return: XForm
"""
return self.xform
@classmethod
def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
try:
instance = cls.objects.get(id=instance_id)
except cls.DoesNotExist:
pass
else:
instance.set_deleted(deleted_at)
def _check_active(self, force):
"""Check that form is active and raise exception if not.
:param force: Ignore restrictions on saving.
"""
if not force and self.xform and not self.xform.downloadable:
raise FormInactiveError()
def _set_geom(self):
xform = self.xform
data_dictionary = xform.data_dictionary()
geo_xpaths = data_dictionary.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
geometry = [float(s) for s in doc.get(xpath, u'').split()]
if len(geometry):
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
doc = self.get_dict()
if not self.date_created:
now = submission_time()
self.date_created = now
point = self.point
if point:
doc[GEOLOCATION] = [point.y, point.x]
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
doc[SUBMITTED_BY] = self.user.username\
if self.user is not None else None
self.json = doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml,
self.xform.data_dictionary())
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def _populate_xml_hash(self):
'''
Populate the `xml_hash` attribute of this `Instance` based on the content of the `xml`
attribute.
'''
self.xml_hash = self.get_hash(self.xml)
@classmethod
def populate_xml_hashes_for_instances(cls,
usernames=None,
pk__in=None,
repopulate=False):
'''
Populate the `xml_hash` field for `Instance` instances limited to the specified users
and/or DB primary keys.
:param list[str] usernames: Optional list of usernames for whom `Instance`s will be
populated with hashes.
:param list[int] pk__in: Optional list of primary keys for `Instance`s that should be
populated with hashes.
:param bool repopulate: Optional argument to force repopulation of existing hashes.
:returns: Total number of `Instance`s updated.
:rtype: int
'''
filter_kwargs = dict()
if usernames:
filter_kwargs['xform__user__username__in'] = usernames
if pk__in:
filter_kwargs['pk__in'] = pk__in
# By default, skip over instances previously populated with hashes.
if not repopulate:
filter_kwargs['xml_hash'] = cls.DEFAULT_XML_HASH
# Query for the target `Instance`s.
target_instances_queryset = cls.objects.filter(**filter_kwargs)
# Exit quickly if there's nothing to do.
if not target_instances_queryset.exists():
return 0
# Limit our queryset result content since we'll only need the `pk` and `xml` attributes.
target_instances_queryset = target_instances_queryset.only('pk', 'xml')
instances_updated_total = 0
# Break the potentially large `target_instances_queryset` into chunks to avoid memory
# exhaustion.
chunk_size = 2000
target_instances_queryset = target_instances_queryset.order_by('pk')
target_instances_qs_chunk = target_instances_queryset
while target_instances_qs_chunk.exists():
# Take a chunk of the target `Instance`s.
target_instances_qs_chunk = target_instances_qs_chunk[0:chunk_size]
for instance in target_instances_qs_chunk:
pk = instance.pk
xml = instance.xml
# Do a `Queryset.update()` on this individual instance to avoid signals triggering
# things like `Reversion` versioning.
instances_updated_count = Instance.objects.filter(
pk=pk).update(xml_hash=cls.get_hash(xml))
instances_updated_total += instances_updated_count
# Set up the next chunk
target_instances_qs_chunk = target_instances_queryset.filter(
pk__gt=instance.pk)
return instances_updated_total
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
return self._parser.get_flat_dict_with_attributes() if flat else\
self._parser.to_dict()
def get_full_dict(self):
# TODO should we store all of these in the JSON no matter what?
d = self.json
data = {
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
self.USERFORM_ID:
u'%s_%s' % (self.user.username, self.xform.id_string),
ATTACHMENTS: [a.media_file.name for a in self.attachments.all()],
self.STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes()
}
if isinstance(self.instance.deleted_at, datetime):
data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
d.update(data)
return d
def get_notes(self):
return [note['note'] for note in self.notes.values('note')]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
@staticmethod
def get_hash(input_string):
'''
Compute the SHA256 hash of the given string. A wrapper to standardize hash computation.
:param basestring input_sting: The string to be hashed.
:return: The resulting hash.
:rtype: str
'''
if isinstance(input_string, unicode):
input_string = input_string.encode('utf-8')
return sha256(input_string).hexdigest()
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def save(self, *args, **kwargs):
force = kwargs.get('force')
if force:
del kwargs['force']
self._check_active(force)
self._set_geom()
self._set_json()
self._set_survey_type()
self._set_uuid()
self._populate_xml_hash()
# Force validation_status to be dict
if self.validation_status is None:
self.validation_status = {}
super(Instance, self).save(*args, **kwargs)
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
# force submission count re-calculation
self.xform.submission_count(force_update=True)
self.parsed_instance.save()
def get_validation_status(self):
"""
Returns instance validation status.
:return: object
"""
# This method can be tweaked to implement default validation status
# For example:
# if not self.validation_status:
# self.validation_status = self.asset.settings.get("validation_statuses")[0]
return self.validation_status
class Instance(models.Model):
json = JSONField(default={}, null=False)
xml = models.TextField()
user = models.ForeignKey(User, related_name='instances', null=True)
xform = models.ForeignKey(XForm, null=True, related_name='instances')
survey_type = models.ForeignKey(SurveyType)
# shows when we first received this instance
date_created = models.DateTimeField(auto_now_add=True)
# this will end up representing "date last parsed"
date_modified = models.DateTimeField(auto_now=True)
# this will end up representing "date instance was deleted"
deleted_at = models.DateTimeField(null=True, default=None)
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
# we add a fourth status: submitted_via_web
status = models.CharField(max_length=20, default=u'submitted_via_web')
uuid = models.CharField(max_length=249, default=u'')
version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True)
# store an geographic objects associated with this instance
geom = models.GeometryCollectionField(null=True)
objects = models.GeoManager()
tags = TaggableManager()
class Meta:
app_label = 'logger'
@classmethod
def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
try:
instance = cls.objects.get(id=instance_id)
except cls.DoesNotExist:
pass
else:
instance.set_deleted(deleted_at)
def numeric_converter(self, json_dict, numeric_fields=None):
if numeric_fields is None:
numeric_fields = get_numeric_fields(self.xform)
for key, value in json_dict.items():
if isinstance(value, basestring) and key in numeric_fields:
converted_value = numeric_checker(value)
if converted_value:
json_dict[key] = converted_value
elif isinstance(value, dict):
json_dict[key] = self.numeric_converter(value, numeric_fields)
elif isinstance(value, list):
for k, v in enumerate(value):
if isinstance(v, basestring) and key in numeric_fields:
converted_value = numeric_checker(v)
if converted_value:
json_dict[key] = converted_value
elif isinstance(v, dict):
value[k] = self.numeric_converter(v, numeric_fields)
return json_dict
def _check_active(self, force):
"""Check that form is active and raise exception if not.
:param force: Ignore restrictions on saving.
"""
if not force and self.xform and not self.xform.downloadable:
raise FormInactiveError()
def _set_geom(self):
xform = self.xform
data_dictionary = xform.data_dictionary()
geo_xpaths = data_dictionary.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
geometry = [float(s) for s in doc.get(xpath, u'').split()]
if len(geometry):
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
self.json = self.get_full_dict()
def get_full_dict(self):
doc = self.json or {}
doc.update(self.get_dict())
if self.id:
doc.update({
UUID:
self.uuid,
ID:
self.id,
BAMBOO_DATASET_ID:
self.xform.bamboo_dataset,
ATTACHMENTS:
_get_attachments_from_instance(self),
STATUS:
self.status,
TAGS:
list(self.tags.names()),
NOTES:
self.get_notes(),
VERSION:
self.version,
DURATION:
self.get_duration(),
XFORM_ID_STRING:
self._parser.get_xform_id_string(),
GEOLOCATION:
[self.point.y, self.point.x] if self.point else [None, None],
SUBMITTED_BY:
self.user.username if self.user else None
})
if isinstance(self.deleted_at, datetime):
doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
if not self.date_created:
self.date_created = submission_time()
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
return doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml,
self.xform.data_dictionary())
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
else self._parser.to_dict()
return self.numeric_converter(instance_dict)
def get_notes(self):
return [note['note'] for note in self.notes.values('note')]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def save(self, *args, **kwargs):
force = kwargs.get('force')
if force:
del kwargs['force']
self._check_active(force)
self._set_geom()
self._set_json()
self._set_survey_type()
self._set_uuid()
self.version = self.xform.version
super(Instance, self).save(*args, **kwargs)
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
# force submission count re-calculation
self.xform.submission_count(force_update=True)
self.parsed_instance.save()
def get_duration(self):
data = self.get_dict()
dd = self.xform.data_dictionary()
start_name = _get_tag_or_element_type_xpath(dd, START)
end_name = _get_tag_or_element_type_xpath(dd, END)
start_time, end_time = data.get(start_name), data.get(end_name)
return calculate_duration(start_time, end_time)
class Instance(models.Model):
json = JSONField(default={}, null=False)
xml = models.TextField()
user = models.ForeignKey(User, related_name='instances', null=True)
xform = models.ForeignKey(XForm, null=True, related_name='instances')
survey_type = models.ForeignKey(SurveyType)
# shows when we first received this instance
date_created = models.DateTimeField(auto_now_add=True)
# this will end up representing "date last parsed"
date_modified = models.DateTimeField(auto_now=True)
# this will end up representing "date instance was deleted"
deleted_at = models.DateTimeField(null=True, default=None)
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
# we add a fourth status: submitted_via_web
status = models.CharField(max_length=20,
default=u'submitted_via_web')
uuid = models.CharField(max_length=249, default=u'')
# store an geographic objects associated with this instance
geom = models.GeometryCollectionField(null=True)
objects = models.GeoManager()
tags = TaggableManager()
class Meta:
app_label = 'logger'
@classmethod
def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
try:
instance = cls.objects.get(id=instance_id)
except cls.DoesNotExist:
pass
else:
instance.set_deleted(deleted_at)
def _check_active(self, force):
"""Check that form is active and raise exception if not.
:param force: Ignore restrictions on saving.
"""
if not force and self.xform and not self.xform.downloadable:
raise FormInactiveError()
def _set_geom(self):
xform = self.xform
data_dictionary = xform.data_dictionary()
geo_xpaths = data_dictionary.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
geometry = [float(s) for s in doc.get(xpath, u'').split()]
if len(geometry):
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
doc = self.get_dict()
if not self.date_created:
now = submission_time()
self.date_created = now
point = self.point
if point:
doc[GEOLOCATION] = [point.y, point.x]
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
doc[SUBMITTED_BY] = self.user.username\
if self.user is not None else None
self.json = doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(
self.xml, self.xform.data_dictionary())
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
return self._parser.get_flat_dict_with_attributes() if flat else\
self._parser.to_dict()
def get_full_dict(self):
# TODO should we store all of these in the JSON no matter what?
d = self.json
data = {
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
self.USERFORM_ID: u'%s_%s' % (
self.user.username,
self.xform.id_string),
ATTACHMENTS: [a.media_file.name for a in
self.attachments.all()],
self.STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes()
}
if isinstance(self.instance.deleted_at, datetime):
data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
d.update(data)
return d
def get_notes(self):
return [note['note'] for note in self.notes.values('note')]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def save(self, *args, **kwargs):
force = kwargs.get('force')
if force:
del kwargs['force']
self._check_active(force)
self._set_geom()
self._set_json()
self._set_survey_type()
self._set_uuid()
super(Instance, self).save(*args, **kwargs)
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
# force submission count re-calculation
self.xform.submission_count(force_update=True)
self.parsed_instance.save()
class Instance(models.Model):
json = JSONField(default={}, null=False)
xml = models.TextField()
user = models.ForeignKey(User, related_name="instances", null=True)
xform = models.ForeignKey(XForm, null=True, related_name="instances")
survey_type = models.ForeignKey(SurveyType)
# shows when we first received this instance
date_created = models.DateTimeField(auto_now_add=True)
# this will end up representing "date last parsed"
date_modified = models.DateTimeField(auto_now=True)
# this will end up representing "date instance was deleted"
deleted_at = models.DateTimeField(null=True, default=None)
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
# we add a fourth status: submitted_via_web
status = models.CharField(max_length=20, default=u"submitted_via_web")
uuid = models.CharField(max_length=249, default=u"")
version = models.CharField(max_length=XFORM_TITLE_LENGTH, null=True)
# store an geographic objects associated with this instance
geom = models.GeometryCollectionField(null=True)
objects = models.GeoManager()
tags = TaggableManager()
class Meta:
app_label = "logger"
@classmethod
def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
try:
instance = cls.objects.get(id=instance_id)
except cls.DoesNotExist:
pass
else:
instance.set_deleted(deleted_at)
def numeric_converter(self, json_dict, numeric_fields=None):
if numeric_fields is None:
numeric_fields = get_numeric_fields(self.xform)
for key, value in json_dict.items():
if isinstance(value, basestring) and key in numeric_fields:
converted_value = numeric_checker(value)
if converted_value:
json_dict[key] = converted_value
elif isinstance(value, dict):
json_dict[key] = self.numeric_converter(value, numeric_fields)
elif isinstance(value, list):
for k, v in enumerate(value):
if isinstance(v, basestring) and key in numeric_fields:
converted_value = numeric_checker(v)
if converted_value:
json_dict[key] = converted_value
elif isinstance(v, dict):
value[k] = self.numeric_converter(v, numeric_fields)
return json_dict
def _check_active(self, force):
"""Check that form is active and raise exception if not.
:param force: Ignore restrictions on saving.
"""
if not force and self.xform and not self.xform.downloadable:
raise FormInactiveError()
def _set_geom(self):
xform = self.xform
data_dictionary = xform.data_dictionary()
geo_xpaths = data_dictionary.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
geometry = [float(s) for s in doc.get(xpath, u"").split()]
if len(geometry):
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
self.json = self.get_full_dict()
def get_full_dict(self):
doc = self.json or {}
doc.update(self.get_dict())
if self.id:
doc.update(
{
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
ATTACHMENTS: _get_attachments_from_instance(self),
STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes(),
VERSION: self.version,
DURATION: self.get_duration(),
XFORM_ID_STRING: self._parser.get_xform_id_string(),
GEOLOCATION: [self.point.y, self.point.x] if self.point else [None, None],
SUBMITTED_BY: self.user.username if self.user else None,
}
)
if isinstance(self.deleted_at, datetime):
doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
if not self.date_created:
self.date_created = submission_time()
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
return doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(self.xml, self.xform.data_dictionary())
def _set_survey_type(self):
self.survey_type, created = SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
instance_dict = self._parser.get_flat_dict_with_attributes() if flat else self._parser.to_dict()
return self.numeric_converter(instance_dict)
def get_notes(self):
return [note["note"] for note in self.notes.values("note")]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def save(self, *args, **kwargs):
force = kwargs.get("force")
if force:
del kwargs["force"]
self._check_active(force)
self._set_geom()
self._set_json()
self._set_survey_type()
self._set_uuid()
self.version = self.xform.version
super(Instance, self).save(*args, **kwargs)
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
# force submission count re-calculation
self.xform.submission_count(force_update=True)
self.parsed_instance.save()
def get_duration(self):
data = self.get_dict()
dd = self.xform.data_dictionary()
start_name = _get_tag_or_element_type_xpath(dd, START)
end_name = _get_tag_or_element_type_xpath(dd, END)
start_time, end_time = data.get(start_name), data.get(end_name)
return calculate_duration(start_time, end_time)
def _set_parser(self):
if not hasattr(self, "_parser"):
# pylint: disable=no-member
self._parser = XFormInstanceParser(self.xml, self.xform)
class InstanceBaseClass(object):
"""Interface of functions for Instance and InstanceHistory model"""
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def numeric_converter(self, json_dict, numeric_fields=None):
if numeric_fields is None:
# pylint: disable=no-member
numeric_fields = get_numeric_fields(self.xform)
for key, value in json_dict.items():
if isinstance(value, basestring) and key in numeric_fields:
converted_value = numeric_checker(value)
if converted_value:
json_dict[key] = converted_value
elif isinstance(value, dict):
json_dict[key] = self.numeric_converter(
value, numeric_fields)
elif isinstance(value, list):
for k, v in enumerate(value):
if isinstance(v, basestring) and key in numeric_fields:
converted_value = numeric_checker(v)
if converted_value:
json_dict[key] = converted_value
elif isinstance(v, dict):
value[k] = self.numeric_converter(
v, numeric_fields)
return json_dict
def _set_geom(self):
# pylint: disable=no-member
xform = self.xform
geo_xpaths = xform.geopoint_xpaths()
doc = self.get_dict()
points = []
if geo_xpaths:
for xpath in geo_xpaths:
for gps in get_values_matching_key(doc, xpath):
try:
geometry = [float(s) for s in gps.split()]
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
except ValueError:
return
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
self.json = self.get_full_dict()
def get_full_dict(self, load_existing=True):
doc = self.json or {} if load_existing else {}
# Get latest dict
doc = self.get_dict()
# pylint: disable=no-member
if self.id:
doc.update({
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
ATTACHMENTS: _get_attachments_from_instance(self),
STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes(),
VERSION: self.version,
DURATION: self.get_duration(),
XFORM_ID_STRING: self._parser.get_xform_id_string(),
XFORM_ID: self.xform.pk,
GEOLOCATION: [self.point.y, self.point.x] if self.point
else [None, None],
SUBMITTED_BY: self.user.username if self.user else None
})
for osm in self.osm_data.all():
doc.update(osm.get_tags_with_prefix())
if isinstance(self.deleted_at, datetime):
doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
# pylint: disable=no-member
if self.has_a_review:
status, comment = self.get_review_status_and_comment()
doc[REVIEW_STATUS] = status
if comment:
doc[REVIEW_COMMENT] = comment
# pylint: disable=attribute-defined-outside-init
if not self.date_created:
self.date_created = submission_time()
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[TOTAL_MEDIA] = self.total_media
doc[MEDIA_COUNT] = self.media_count
doc[MEDIA_ALL_RECEIVED] = self.media_all_received
edited = False
if hasattr(self, 'last_edited'):
edited = self.last_edited is not None
doc[EDITED] = edited
edited and doc.update({
LAST_EDITED: convert_to_serializable_date(self.last_edited)
})
return doc
def _set_parser(self):
if not hasattr(self, "_parser"):
# pylint: disable=no-member
self._parser = XFormInstanceParser(self.xml, self.xform)
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
# pylint: disable=no-member, attribute-defined-outside-init
if self.xml and not self.uuid:
# pylint: disable=no-member
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
else self._parser.to_dict()
return self.numeric_converter(instance_dict)
def get_notes(self):
# pylint: disable=no-member
return [note.get_data() for note in self.notes.all()]
def get_review_status_and_comment(self):
"""
Return a tuple of review status and comment
"""
try:
# pylint: disable=no-member
status = self.reviews.latest('date_modified').status
comment = self.reviews.latest('date_modified').get_note_text()
return status, comment
except SubmissionReview.DoesNotExist:
return None
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
def get_duration(self):
data = self.get_dict()
# pylint: disable=no-member
start_name = _get_tag_or_element_type_xpath(self.xform, START)
end_name = _get_tag_or_element_type_xpath(self.xform, END)
start_time, end_time = data.get(start_name), data.get(end_name)
return calculate_duration(start_time, end_time)
class InstanceBaseClass(object):
"""Interface of functions for Instance and InstanceHistory model"""
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def numeric_converter(self, json_dict, numeric_fields=None):
if numeric_fields is None:
# pylint: disable=no-member
numeric_fields = get_numeric_fields(self.xform)
for key, value in json_dict.items():
if isinstance(value, basestring) and key in numeric_fields:
converted_value = numeric_checker(value)
if converted_value:
json_dict[key] = converted_value
elif isinstance(value, dict):
json_dict[key] = self.numeric_converter(
value, numeric_fields)
elif isinstance(value, list):
for k, v in enumerate(value):
if isinstance(v, basestring) and key in numeric_fields:
converted_value = numeric_checker(v)
if converted_value:
json_dict[key] = converted_value
elif isinstance(v, dict):
value[k] = self.numeric_converter(
v, numeric_fields)
return json_dict
def _set_geom(self):
# pylint: disable=no-member
xform = self.xform
geo_xpaths = xform.geopoint_xpaths()
doc = self.get_dict()
points = []
if geo_xpaths:
for xpath in geo_xpaths:
for gps in get_values_matching_key(doc, xpath):
try:
geometry = [float(s) for s in gps.split()]
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
except ValueError:
return
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
self.json = self.get_full_dict()
def get_full_dict(self, load_existing=True):
doc = self.json or {} if load_existing else {}
# Get latest dict
doc = self.get_dict()
# pylint: disable=no-member
if self.id:
doc.update({
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
ATTACHMENTS: _get_attachments_from_instance(self),
STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes(),
VERSION: self.version,
DURATION: self.get_duration(),
XFORM_ID_STRING: self._parser.get_xform_id_string(),
XFORM_ID: self.xform.pk,
GEOLOCATION: [self.point.y, self.point.x] if self.point
else [None, None],
SUBMITTED_BY: self.user.username if self.user else None
})
for osm in self.osm_data.all():
doc.update(osm.get_tags_with_prefix())
if isinstance(self.deleted_at, datetime):
doc[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
# pylint: disable=no-member
if self.has_a_review:
review = self.get_latest_review()
if review:
doc[REVIEW_STATUS] = review.status
doc[REVIEW_DATE] = review.date_created.strftime(
MONGO_STRFTIME)
if review.get_note_text():
doc[REVIEW_COMMENT] = review.get_note_text()
# pylint: disable=attribute-defined-outside-init
if not self.date_created:
self.date_created = submission_time()
if not self.date_modified:
self.date_modified = self.date_created
doc[DATE_MODIFIED] = self.date_modified.strftime(
MONGO_STRFTIME)
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[TOTAL_MEDIA] = self.total_media
doc[MEDIA_COUNT] = self.media_count
doc[MEDIA_ALL_RECEIVED] = self.media_all_received
edited = False
if hasattr(self, 'last_edited'):
edited = self.last_edited is not None
doc[EDITED] = edited
edited and doc.update({
LAST_EDITED: convert_to_serializable_date(self.last_edited)
})
return doc
def _set_parser(self):
if not hasattr(self, "_parser"):
# pylint: disable=no-member
self._parser = XFormInstanceParser(self.xml, self.xform)
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
# pylint: disable=no-member, attribute-defined-outside-init
if self.xml and not self.uuid:
# pylint: disable=no-member
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
instance_dict = self._parser.get_flat_dict_with_attributes() if flat \
else self._parser.to_dict()
return self.numeric_converter(instance_dict)
def get_notes(self):
# pylint: disable=no-member
return [note.get_data() for note in self.notes.all()]
@deprecated(version='2.5.3',
reason="Deprecated in favour of `get_latest_review`")
def get_review_status_and_comment(self):
"""
Return a tuple of review status and comment.
"""
try:
# pylint: disable=no-member
status = self.reviews.latest('date_modified').status
comment = self.reviews.latest('date_modified').get_note_text()
return status, comment
except SubmissionReview.DoesNotExist:
return None
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
def get_duration(self):
data = self.get_dict()
# pylint: disable=no-member
start_name = _get_tag_or_element_type_xpath(self.xform, START)
end_name = _get_tag_or_element_type_xpath(self.xform, END)
start_time, end_time = data.get(start_name), data.get(end_name)
return calculate_duration(start_time, end_time)
def get_latest_review(self):
"""
Returns the latest review.
Used in favour of `get_review_status_and_comment`.
"""
try:
return self.reviews.latest('date_modified')
except SubmissionReview.DoesNotExist:
return None
class Instance(models.Model):
XML_HASH_LENGTH = 64
DEFAULT_XML_HASH = None
json = JSONField(default={}, null=False)
xml = models.TextField()
xml_hash = models.CharField(max_length=XML_HASH_LENGTH, db_index=True, null=True,
default=DEFAULT_XML_HASH)
user = models.ForeignKey(User, related_name='instances', null=True)
xform = models.ForeignKey(XForm, null=True, related_name='instances')
survey_type = models.ForeignKey(SurveyType)
# shows when we first received this instance
date_created = models.DateTimeField(auto_now_add=True)
# this will end up representing "date last parsed"
date_modified = models.DateTimeField(auto_now=True)
# this will end up representing "date instance was deleted"
deleted_at = models.DateTimeField(null=True, default=None)
# ODK keeps track of three statuses for an instance:
# incomplete, submitted, complete
# we add a fourth status: submitted_via_web
status = models.CharField(max_length=20,
default=u'submitted_via_web')
uuid = models.CharField(max_length=249, default=u'', db_index=True)
# store an geographic objects associated with this instance
geom = models.GeometryCollectionField(null=True)
objects = models.GeoManager()
tags = TaggableManager()
validation_status = JSONField(null=True, default=None)
# TODO Don't forget to update all records with command `update_is_sync_with_mongo`.
is_synced_with_mongo = LazyDefaultBooleanField(default=False)
# If XForm.has_kpi_hooks` is True, this field should be True either.
# It tells whether the instance has been successfully sent to KPI.
posted_to_kpi = LazyDefaultBooleanField(default=False)
class Meta:
app_label = 'logger'
@property
def asset(self):
"""
The goal of this property is to make the code future proof.
We can run the tests on kpi backend or kobocat backend.
Instance.asset will exist for both
It's used for validation_statuses.
:return: XForm
"""
return self.xform
@classmethod
def set_deleted_at(cls, instance_id, deleted_at=timezone.now()):
try:
instance = cls.objects.get(id=instance_id)
except cls.DoesNotExist:
pass
else:
instance.set_deleted(deleted_at)
def _check_active(self, force):
"""Check that form is active and raise exception if not.
:param force: Ignore restrictions on saving.
"""
if not force and self.xform and not self.xform.downloadable:
raise FormInactiveError()
def _set_geom(self):
xform = self.xform
data_dictionary = xform.data_dictionary()
geo_xpaths = data_dictionary.geopoint_xpaths()
doc = self.get_dict()
points = []
if len(geo_xpaths):
for xpath in geo_xpaths:
geometry = [float(s) for s in doc.get(xpath, u'').split()]
if len(geometry):
lat, lng = geometry[0:2]
points.append(Point(lng, lat))
if not xform.instances_with_geopoints and len(points):
xform.instances_with_geopoints = True
xform.save()
self.geom = GeometryCollection(points)
def _set_json(self):
doc = self.get_dict()
if not self.date_created:
now = submission_time()
self.date_created = now
point = self.point
if point:
doc[GEOLOCATION] = [point.y, point.x]
doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
doc[XFORM_ID_STRING] = self._parser.get_xform_id_string()
doc[SUBMITTED_BY] = self.user.username\
if self.user is not None else None
self.json = doc
def _set_parser(self):
if not hasattr(self, "_parser"):
self._parser = XFormInstanceParser(
self.xml, self.xform.data_dictionary())
def _set_survey_type(self):
self.survey_type, created = \
SurveyType.objects.get_or_create(slug=self.get_root_node_name())
def _set_uuid(self):
if self.xml and not self.uuid:
uuid = get_uuid_from_xml(self.xml)
if uuid is not None:
self.uuid = uuid
set_uuid(self)
def _populate_xml_hash(self):
'''
Populate the `xml_hash` attribute of this `Instance` based on the content of the `xml`
attribute.
'''
self.xml_hash = self.get_hash(self.xml)
@classmethod
def populate_xml_hashes_for_instances(cls, usernames=None, pk__in=None, repopulate=False):
'''
Populate the `xml_hash` field for `Instance` instances limited to the specified users
and/or DB primary keys.
:param list[str] usernames: Optional list of usernames for whom `Instance`s will be
populated with hashes.
:param list[int] pk__in: Optional list of primary keys for `Instance`s that should be
populated with hashes.
:param bool repopulate: Optional argument to force repopulation of existing hashes.
:returns: Total number of `Instance`s updated.
:rtype: int
'''
filter_kwargs = dict()
if usernames:
filter_kwargs['xform__user__username__in'] = usernames
if pk__in:
filter_kwargs['pk__in'] = pk__in
# By default, skip over instances previously populated with hashes.
if not repopulate:
filter_kwargs['xml_hash'] = cls.DEFAULT_XML_HASH
# Query for the target `Instance`s.
target_instances_queryset = cls.objects.filter(**filter_kwargs)
# Exit quickly if there's nothing to do.
if not target_instances_queryset.exists():
return 0
# Limit our queryset result content since we'll only need the `pk` and `xml` attributes.
target_instances_queryset = target_instances_queryset.only('pk', 'xml')
instances_updated_total = 0
# Break the potentially large `target_instances_queryset` into chunks to avoid memory
# exhaustion.
chunk_size = 2000
target_instances_queryset = target_instances_queryset.order_by('pk')
target_instances_qs_chunk = target_instances_queryset
while target_instances_qs_chunk.exists():
# Take a chunk of the target `Instance`s.
target_instances_qs_chunk = target_instances_qs_chunk[0:chunk_size]
for instance in target_instances_qs_chunk:
pk = instance.pk
xml = instance.xml
# Do a `Queryset.update()` on this individual instance to avoid signals triggering
# things like `Reversion` versioning.
instances_updated_count = Instance.objects.filter(pk=pk).update(
xml_hash=cls.get_hash(xml))
instances_updated_total += instances_updated_count
# Set up the next chunk
target_instances_qs_chunk = target_instances_queryset.filter(
pk__gt=instance.pk)
return instances_updated_total
def get(self, abbreviated_xpath):
self._set_parser()
return self._parser.get(abbreviated_xpath)
def get_dict(self, force_new=False, flat=True):
"""Return a python object representation of this instance's XML."""
self._set_parser()
return self._parser.get_flat_dict_with_attributes() if flat else\
self._parser.to_dict()
def get_full_dict(self):
# TODO should we store all of these in the JSON no matter what?
d = self.json
data = {
UUID: self.uuid,
ID: self.id,
BAMBOO_DATASET_ID: self.xform.bamboo_dataset,
self.USERFORM_ID: u'%s_%s' % (
self.user.username,
self.xform.id_string),
ATTACHMENTS: [a.media_file.name for a in
self.attachments.all()],
self.STATUS: self.status,
TAGS: list(self.tags.names()),
NOTES: self.get_notes()
}
if isinstance(self.instance.deleted_at, datetime):
data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME)
d.update(data)
return d
def get_notes(self):
return [note['note'] for note in self.notes.values('note')]
def get_root_node(self):
self._set_parser()
return self._parser.get_root_node()
def get_root_node_name(self):
self._set_parser()
return self._parser.get_root_node_name()
@staticmethod
def get_hash(input_string):
'''
Compute the SHA256 hash of the given string. A wrapper to standardize hash computation.
:param basestring input_sting: The string to be hashed.
:return: The resulting hash.
:rtype: str
'''
if isinstance(input_string, unicode):
input_string = input_string.encode('utf-8')
return sha256(input_string).hexdigest()
@property
def point(self):
gc = self.geom
if gc and len(gc):
return gc[0]
def save(self, *args, **kwargs):
force = kwargs.pop("force", False)
self._check_active(force)
self._set_geom()
self._set_json()
self._set_survey_type()
self._set_uuid()
self._populate_xml_hash()
# Force validation_status to be dict
if self.validation_status is None:
self.validation_status = {}
super(Instance, self).save(*args, **kwargs)
def set_deleted(self, deleted_at=timezone.now()):
self.deleted_at = deleted_at
self.save()
# force submission count re-calculation
self.xform.submission_count(force_update=True)
self.parsed_instance.save()
def get_validation_status(self):
"""
Returns instance validation status.
:return: object
"""
# This method can be tweaked to implement default validation status
# For example:
# if not self.validation_status:
# self.validation_status = self.asset.settings.get("validation_statuses")[0]
return self.validation_status
