def setUp(self):
self.user = User.objects.create(username="******",
email="*****@*****.**")
self.project = UserProfile.objects.create(user=self.user)
self.user.set_password("pass")
self.project = get_user_default_project(self.user)
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.project = self.project
self.xform1.json = '{"id_string": "yes_or_no_form", "children": '\
'[{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.project = self.project
self.xform2.json = '{"id_string": "start_time_form", "children": '\
'[{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime_form", "title": "start_time_form",'\
'"type": "survey"}'\
.strip()
self._get_xml_for_form(self.xform1)
self._get_xml_for_form(self.xform2)
def setUp(self):
self.user = User.objects.create(username="******",
email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
def get_xml_for_form(xform):
builder = SurveyElementBuilder()
sss = builder.create_survey_element_from_json(xform.json)
xform.xml = sss.to_xml()
xform._mark_start_time_boolean()
xform.save()
get_xml_for_form(self.xform1)
get_xml_for_form(self.xform2)
def publish_xml_form(xml_file, user, project, id_string=None, created_by=None):
xml = xml_file.read()
if isinstance(xml, bytes):
xml = xml.decode('utf-8')
survey = create_survey_element_from_xml(xml)
form_json = survey.to_json()
if id_string:
dd = DataDictionary.objects.get(user=user,
id_string=id_string,
project=project)
dd.xml = xml
dd.json = form_json
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml()
dd._set_hash()
dd.save()
return dd
else:
created_by = created_by or user
dd = DataDictionary(created_by=created_by,
user=user,
xml=xml,
json=form_json,
project=project)
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml(file_name=xml_file.name)
dd._set_hash()
dd.save()
return dd
def publish_xml_form(xml_file, user, project, id_string=None, created_by=None):
xml = xml_file.read()
if isinstance(xml, bytes):
xml = xml.decode('utf-8')
survey = create_survey_element_from_xml(xml)
form_json = survey.to_json()
if id_string:
dd = DataDictionary.objects.get(
user=user, id_string=id_string, project=project)
dd.xml = xml
dd.json = form_json
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml()
dd._set_hash()
dd.save()
return dd
else:
created_by = created_by or user
dd = DataDictionary(
created_by=created_by,
user=user,
xml=xml,
json=form_json,
project=project)
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml(file_name=xml_file.name)
dd._set_hash()
dd.save()
return dd
def setUp(self):
self.user = User.objects.create(
username="******", email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
def get_xml_for_form(xform):
builder = SurveyElementBuilder()
sss = builder.create_survey_element_from_json(xform.json)
xform.xml = sss.to_xml()
xform._mark_start_time_boolean()
xform.save()
get_xml_for_form(self.xform1)
get_xml_for_form(self.xform2)
def publish_xml_form(xml_file, user, id_string=None):
xml = xml_file.read()
survey = create_survey_element_from_xml(xml)
form_json = survey.to_json()
if id_string:
dd = DataDictionary.objects.get(user=user, id_string=id_string)
dd.xml = xml
dd.json = form_json
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml()
dd.save()
return dd
else:
dd = DataDictionary(user=user, xml=xml, json=form_json)
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml(file_name=xml_file.name)
dd.save()
return dd
def setUp(self):
self.user = User.objects.create(
username="******", email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
self._get_xml_for_form(self.xform1)
self._get_xml_for_form(self.xform2)
def _split_gps_fields(cls, record, gps_fields):
updated_gps_fields = {}
for key, value in record.iteritems():
if key in gps_fields and isinstance(value, basestring):
gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key)
gps_parts = dict([(xpath, None) for xpath in gps_xpaths])
# hack, check if its a list and grab the object within that
parts = value.split(' ')
# TODO: check whether or not we can have a gps recording
# from ODKCollect that has less than four components,
# for now we are assuming that this is not the case.
if len(parts) == 4:
gps_parts = dict(zip(gps_xpaths, parts))
updated_gps_fields.update(gps_parts)
# check for repeats within record i.e. in value
elif type(value) == list:
for list_item in value:
if type(list_item) == dict:
cls._split_gps_fields(list_item, gps_fields)
record.update(updated_gps_fields)
def _split_gps_fields(cls, record, gps_fields):
updated_gps_fields = {}
for key, value in record.iteritems():
if key in gps_fields and isinstance(value, basestring):
gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key)
gps_parts = dict([(xpath, None) for xpath in gps_xpaths])
# hack, check if its a list and grab the object within that
parts = value.split(' ')
# TODO: check whether or not we can have a gps recording
# from ODKCollect that has less than four components,
# for now we are assuming that this is not the case.
if len(parts) == 4:
gps_parts = dict(zip(gps_xpaths, parts))
updated_gps_fields.update(gps_parts)
# check for repeats within record i.e. in value
elif type(value) == list:
for list_item in value:
if type(list_item) == dict:
cls._split_gps_fields(list_item, gps_fields)
record.update(updated_gps_fields)
def publish_xml_form(xml_file, user, id_string=None):
xml = xml_file.read()
survey = create_survey_element_from_xml(xml)
form_json = survey.to_json()
if id_string:
dd = DataDictionary.objects.get(user=user, id_string=id_string)
dd.xml = xml
dd.json = form_json
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml()
dd.save()
return dd
else:
dd = DataDictionary(user=user, xml=xml, json=form_json)
dd._mark_start_time_boolean()
set_uuid(dd)
dd._set_uuid_in_xml(file_name=xml_file.name)
dd.save()
return dd
def get_data_dictionary_from_survey(survey):
dd = DataDictionary()
dd._survey = survey
return dd
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/',
remove_group_name=False):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter,
remove_group_name)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter,
remove_group_name)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
child.get_abbreviated_xpath(), field_delimiter, dd,
remove_group_name)
_label = \
dd.get_label(child_xpath, elem=child) or _title
current_section['elements'].append({
'label':
_label,
'title':
_title,
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
choices = self._get_select_mulitples_choices(
child, dd, field_delimiter, remove_group_name)
for choice in choices:
if choice not in current_section['elements']:
current_section['elements'].append(choice)
choices_xpaths = [c['xpath'] for c in choices]
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), choices_xpaths)
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
for xpath in xpaths:
_title = ExportBuilder.format_field_title(
xpath, field_delimiter, dd, remove_group_name)
current_section['elements'].append({
'label':
_title,
'title':
_title,
'xpath':
xpath,
'type':
'decimal'
})
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def process_tableau_data(data, xform):
"""
Streamlines the row header fields
with the column header fields for the same form.
Handles Flattenning repeat data for tableau
"""
def get_xpath(key, nested_key):
val = nested_key.split('/')
nested_key_diff = val[len(key.split('/')):]
xpaths = key + f'[{index}]/' + '/'.join(nested_key_diff)
return xpaths
def get_updated_data_dict(key, value, data_dict):
"""
Generates key, value pairs for select multiple question types.
Defining the new xpaths from the
question name(key) and the choice name(value)
in accordance with how we generate the tableau schema.
"""
if isinstance(value, str) and data_dict:
choices = value.split(" ")
for choice in choices:
xpaths = f'{key}/{choice}'
data_dict[xpaths] = choice
elif isinstance(value, list):
try:
for item in value:
for (nested_key, nested_val) in item.items():
xpath = get_xpath(key, nested_key)
data_dict[xpath] = nested_val
except AttributeError:
data_dict[key] = value
return data_dict
def get_ordered_repeat_value(key, item, index):
"""
Return Ordered Dict of repeats in the order in which they appear in
the XForm.
"""
children = xform.get_child_elements(key, split_select_multiples=False)
item_list = OrderedDict()
data = {}
for elem in children:
if not question_types_to_exclude(elem.type):
new_xpath = elem.get_abbreviated_xpath()
item_list[new_xpath] = item.get(new_xpath, DEFAULT_NA_REP)
# Loop through repeat data and flatten it
# given the key "children/details" and nested_key/
# abbreviated xpath "children/details/immunization/polio_1",
# generate ["children", index, "immunization/polio_1"]
for (nested_key, nested_val) in item_list.items():
qstn_type = xform.get_element(nested_key).type
xpaths = get_xpath(key, nested_key)
if qstn_type == MULTIPLE_SELECT_TYPE:
data = get_updated_data_dict(xpaths, nested_val, data)
elif qstn_type == REPEAT_SELECT_TYPE:
data = get_updated_data_dict(xpaths, nested_val, data)
else:
data[xpaths] = nested_val
return data
result = []
if data:
headers = xform.get_headers()
tableau_headers = remove_metadata_fields(headers)
for row in data:
diff = set(tableau_headers).difference(set(row))
flat_dict = dict.fromkeys(diff, None)
for (key, value) in row.items():
if isinstance(value, list) and key not in [
ATTACHMENTS, NOTES, GEOLOCATION
]:
for index, item in enumerate(value, start=1):
# order repeat according to xform order
item = get_ordered_repeat_value(key, item, index)
flat_dict.update(item)
else:
try:
qstn_type = xform.get_element(key).type
if qstn_type == MULTIPLE_SELECT_TYPE:
flat_dict = get_updated_data_dict(
key, value, flat_dict)
if qstn_type == 'geopoint':
parts = value.split(' ')
gps_xpaths = \
DataDictionary.get_additional_geopoint_xpaths(
key)
gps_parts = dict([(xpath, None)
for xpath in gps_xpaths])
if len(parts) == 4:
gps_parts = dict(zip(gps_xpaths, parts))
flat_dict.update(gps_parts)
else:
flat_dict[key] = value
except AttributeError:
flat_dict[key] = value
result.append(flat_dict)
return result
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
select_one,
select_label_map,
field_delimiter='/'):
print "survey_element.children"
print survey_element.children
print "current_section"
print current_section
for child in survey_element.children:
print "survey_element.child"
print child
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, select_one,
select_label_map, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, select_one,
select_label_map, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
# print('child bind type: ' ,child.bind.get(u"type"))
if child.bind.get(
u"type"
) == SINGLE_SELECT_BIND_TYPE and self.SHOW_LABEL:
# print ('get select 1 type question :D ')
for c in child.children:
_xpath = c.get_abbreviated_xpath()
if not self.SHOW_LABEL:
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
else:
_title = c.label
_title_custom = c.label
select_label_map[_xpath] = _title_custom
# _xpath = c.get_abbreviated_xpath()
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string',
'label': _title
}
# if choice not in current_section['elements']:current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_one,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type"),
'label':
child.label
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
_title_custom = c.label
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string',
'label': _title_custom
}
if self.SHOW_LABEL:
select_label_map[_xpath] = _title_custom
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
class TestSimpleSubmission(TestCase):
def _get_xml_for_form(self, xform):
builder = SurveyElementBuilder()
sss = builder.create_survey_element_from_json(xform.json)
xform.xml = sss.to_xml()
xform._mark_start_time_boolean()
xform.save()
def _submit_at_hour(self, hour):
st_xml = '<?xml version=\'1.0\' ?><start_time id="start_time"><st'\
'art_time>2012-01-11T%d:00:00.000+00</start_time></start'\
'_time>' % hour
try:
create_instance(self.user.username, TempFileProxy(st_xml), [])
except DuplicateInstance:
pass
def _submit_simple_yes(self):
create_instance(self.user.username, TempFileProxy(
'<?xml version=\'1.0\' ?><yes_or_no id="yes_or_no"><yesno>Yes<'
'/yesno></yes_or_no>'), [])
def setUp(self):
self.user = User.objects.create(
username="******", email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
self._get_xml_for_form(self.xform1)
self._get_xml_for_form(self.xform2)
def tearDown(self):
self.xform1.delete()
self.user.delete()
def test_start_time_boolean_properly_set(self):
self.assertFalse(self.xform1.has_start_time)
self.assertTrue(self.xform2.has_start_time)
def test_simple_yes_submission(self):
self.assertEquals(0, self.xform1.instances.count())
self._submit_simple_yes()
self.assertEquals(1, self.xform1.instances.count())
self._submit_simple_yes()
# a simple "yes" submission *SHOULD* increment the survey count
self.assertEquals(2, self.xform1.instances.count())
def test_start_time_submissions(self):
"""This test checks to make sure that instances
*with start_time available* are marked as duplicates when the XML is a
direct match.
"""
self.assertEquals(0, self.xform2.instances.count())
self._submit_at_hour(11)
self.assertEquals(1, self.xform2.instances.count())
self._submit_at_hour(12)
self.assertEquals(2, self.xform2.instances.count())
# an instance from 11 AM already exists in the database, so it
# *SHOULD NOT* increment the survey count.
self._submit_at_hour(11)
self.assertEquals(2, self.xform2.instances.count())
def test_corrupted_submission(self):
"""Test xml submissions that contain unicode characters.
"""
xml = 'v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9\'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^\'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13\'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh\'|?\xff\xff\xff\xff\xd5\'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=' # noqa
request = RequestFactory().post('/')
request.user = self.user
error, instance = safe_create_instance(
self.user.username, TempFileProxy(xml), None, None, request)
text = 'File likely corrupted during transmission'
self.assertContains(error, text, status_code=400)
class TestSimpleSubmission(TestCase):
def setUp(self):
self.user = User.objects.create(
username="******", email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
def get_xml_for_form(xform):
builder = SurveyElementBuilder()
sss = builder.create_survey_element_from_json(xform.json)
xform.xml = sss.to_xml()
xform._mark_start_time_boolean()
xform.save()
get_xml_for_form(self.xform1)
get_xml_for_form(self.xform2)
def tearDown(self):
self.xform1.delete()
self.user.delete()
def test_start_time_boolean_properly_set(self):
self.assertFalse(self.xform1.has_start_time)
self.assertTrue(self.xform2.has_start_time)
def test_simple_yes_submission(self):
def submit_simple_yes():
create_instance(self.user.username, TempFileProxy("""
<?xml version='1.0' ?><yes_or_no id="yes_or_no"><yesno>Yes</yesno></yes_or_no>
""".strip()), [])
self.assertEquals(0, self.xform1.instances.count())
submit_simple_yes()
self.assertEquals(1, self.xform1.instances.count())
# a simple "yes" submission *SHOULD* increment the survey count
submit_simple_yes()
self.assertEquals(2, self.xform1.instances.count())
def test_start_time_submissions(self):
"""
This test checks to make sure that instances
*with start_time available* are marked as duplicates when the XML is a
direct match.
"""
def submit_at_hour(hour):
st_xml = """
<?xml version='1.0' ?><start_time id="start_time"><start_time>2012-01-11T%d:00:00.000+00</start_time></start_time>
""".strip() % hour
try:
create_instance(self.user.username, TempFileProxy(st_xml), [])
except DuplicateInstance:
pass
self.assertEquals(0, self.xform2.instances.count())
submit_at_hour(11)
self.assertEquals(1, self.xform2.instances.count())
submit_at_hour(12)
self.assertEquals(2, self.xform2.instances.count())
# an instance from 11 AM already exists in the database, so it
# *SHOULD NOT* increment the survey count.
submit_at_hour(11)
self.assertEquals(2, self.xform2.instances.count())
def build_sections(
current_section, survey_element, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []}
self.sections.append(section)
build_sections(
section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(
current_section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title': ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath': child_xpath,
'type': child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(),
[c.get_abbreviated_xpath()
for c in child.children])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend(
[
{
'title': ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath': xpath,
'type': 'decimal'
}
for xpath in xpaths
])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
class TestSimpleSubmission(TestCase):
def _get_xml_for_form(self, xform):
builder = SurveyElementBuilder()
sss = builder.create_survey_element_from_json(xform.json)
xform.xml = sss.to_xml()
xform._mark_start_time_boolean()
xform.save()
def _submit_at_hour(self, hour):
st_xml = '<?xml version=\'1.0\' ?><start_time id="start_time"><st'\
'art_time>2012-01-11T%d:00:00.000+00</start_time></start'\
'_time>' % hour
try:
create_instance(self.user.username, TempFileProxy(st_xml), [])
except DuplicateInstance:
pass
def _submit_simple_yes(self):
create_instance(
self.user.username,
TempFileProxy(
'<?xml version=\'1.0\' ?><yes_or_no id="yes_or_no"><yesno>Yes<'
'/yesno></yes_or_no>'), [])
def setUp(self):
self.user = User.objects.create(username="******",
email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
self._get_xml_for_form(self.xform1)
self._get_xml_for_form(self.xform2)
def tearDown(self):
self.xform1.delete()
self.user.delete()
def test_start_time_boolean_properly_set(self):
self.assertFalse(self.xform1.has_start_time)
self.assertTrue(self.xform2.has_start_time)
def test_simple_yes_submission(self):
self.assertEquals(0, self.xform1.instances.count())
self._submit_simple_yes()
self.assertEquals(1, self.xform1.instances.count())
self._submit_simple_yes()
# a simple "yes" submission *SHOULD* increment the survey count
self.assertEquals(2, self.xform1.instances.count())
def test_start_time_submissions(self):
"""This test checks to make sure that instances
*with start_time available* are marked as duplicates when the XML is a
direct match.
"""
self.assertEquals(0, self.xform2.instances.count())
self._submit_at_hour(11)
self.assertEquals(1, self.xform2.instances.count())
self._submit_at_hour(12)
self.assertEquals(2, self.xform2.instances.count())
# an instance from 11 AM already exists in the database, so it
# *SHOULD NOT* increment the survey count.
self._submit_at_hour(11)
self.assertEquals(2, self.xform2.instances.count())
def test_corrupted_submission(self):
"""Test xml submissions that contain unicode characters.
"""
xml = 'v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9\'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^\'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13\'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh\'|?\xff\xff\xff\xff\xd5\'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=' # noqa
request = RequestFactory().post('/')
request.user = self.user
error, instance = safe_create_instance(self.user.username,
TempFileProxy(xml), None, None,
request)
text = 'File likely corrupted during transmission'
self.assertContains(error, text, status_code=400)
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if MongoHelper.is_attribute_invalid(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: MongoHelper.encode(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
class TestSimpleSubmission(TestCase):
def setUp(self):
self.user = User.objects.create(username="******",
email="*****@*****.**")
self.user.set_password("pass")
self.xform1 = DataDictionary()
self.xform1.user = self.user
self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\
'"yesno", "label": "Yes or no?", "type": "text"}],'\
' "name": "yes_or_no", "title": "yes_or_no", "type'\
'": "survey"}'.strip()
self.xform2 = DataDictionary()
self.xform2.user = self.user
self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\
'"start_time", "type": "start"}], "name": "start_t'\
'ime", "title": "start_time", "type": "survey"}'\
.strip()
def get_xml_for_form(xform):
builder = SurveyElementBuilder()
sss = builder.create_survey_element_from_json(xform.json)
xform.xml = sss.to_xml()
xform._mark_start_time_boolean()
xform.save()
get_xml_for_form(self.xform1)
get_xml_for_form(self.xform2)
def tearDown(self):
self.xform1.delete()
self.user.delete()
def test_start_time_boolean_properly_set(self):
self.assertFalse(self.xform1.has_start_time)
self.assertTrue(self.xform2.has_start_time)
def test_simple_yes_submission(self):
def submit_simple_yes():
create_instance(
self.user.username,
TempFileProxy(
'<?xml version=\'1.0\' ?><yes_or_no id="yes_or_no"><yesno>Yes<'
'/yesno></yes_or_no>'), [])
self.assertEquals(0, self.xform1.instances.count())
submit_simple_yes()
self.assertEquals(1, self.xform1.instances.count())
# a simple "yes" submission *SHOULD* increment the survey count
submit_simple_yes()
self.assertEquals(2, self.xform1.instances.count())
def test_start_time_submissions(self):
"""
This test checks to make sure that instances
*with start_time available* are marked as duplicates when the XML is a
direct match.
"""
def submit_at_hour(hour):
st_xml = '<?xml version=\'1.0\' ?><start_time id="start_time"><st'\
'art_time>2012-01-11T%d:00:00.000+00</start_time></start'\
'_time>' % hour
try:
create_instance(self.user.username, TempFileProxy(st_xml), [])
except DuplicateInstance:
pass
self.assertEquals(0, self.xform2.instances.count())
submit_at_hour(11)
self.assertEquals(1, self.xform2.instances.count())
submit_at_hour(12)
self.assertEquals(2, self.xform2.instances.count())
# an instance from 11 AM already exists in the database, so it
# *SHOULD NOT* increment the survey count.
submit_at_hour(11)
self.assertEquals(2, self.xform2.instances.count())
