def test_csv(self):
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
options = {'versions': 'gqs'}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"q1";"g1q1";"g1sg1q1";"g1q2";"g2q1";"qz"
"respondent1's r1";"respondent1's r2";"respondent1's r2.5";"respondent1's r2.75 :)";"respondent1's r3";"respondent1's r4"
"respondent2's r1";"respondent2's r2";"respondent2's r2.5";"respondent2's r2.75 :)";"respondent2's r3";"respondent2's r4"
"""
self.assertTextEqual(csv_data, expected)
options = {'versions': 'gqs', 'hierarchy_in_labels': True}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"q1";"g1/g1q1";"g1/sg1/g1sg1q1";"g1/g1q2";"g2/g2q1";"qz"
"respondent1's r1";"respondent1's r2";"respondent1's r2.5";"respondent1's r2.75 :)";"respondent1's r3";"respondent1's r4"
"respondent2's r1";"respondent2's r2";"respondent2's r2.5";"respondent2's r2.75 :)";"respondent2's r3";"respondent2's r4"
"""
self.assertTextEqual(csv_data, expected)
options = {
'versions': 'gqs',
'hierarchy_in_labels': True,
'lang': UNTRANSLATED
}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"Q1";"Group 1/G1Q1";"Group 1/Sub Group 1/G1SG1Q1";"Group 1/G1Q2";"g2/G2Q1";"QZed"
"respondent1's r1";"respondent1's r2";"respondent1's r2.5";"respondent1's r2.75 :)";"respondent1's r3";"respondent1's r4"
"respondent2's r1";"respondent2's r2";"respondent2's r2.5";"respondent2's r2.75 :)";"respondent2's r3";"respondent2's r4"
"""
self.assertTextEqual(csv_data, expected)
title, schemas, submissions = restaurant_profile
fp = FormPack(schemas, title)
options = {'versions': 'rpV3', 'lang': fp[1].translations[1]}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"nom du restaurant";"lieu";"_lieu_latitude";"_lieu_longitude";"_lieu_altitude";"_lieu_precision";"type de restaurant"
"Taco Truck";"13.42 -25.43";"13.42";"-25.43";"";"";"avec vente à emporter"
"Harvest";"12.43 -24.53";"12.43";"-24.53";"";"";"traditionnel"
"""
self.assertTextEqual(csv_data, expected)
def test_groups_disabled():
scontent = {
'content': {
'survey': [
{
'type': 'text',
'name': 'n1',
'label': ['aa']
},
{
'type': 'begin_group',
'name': 'nada'
},
{
'type': 'note',
'name': 'n2',
'label': ['ab']
},
{
'type': 'end_group'
},
],
'translations': ['en'],
'translated': ['label'],
}
}
(ga, gz) = [scontent['content']['survey'][nn] for nn in [1, 3]]
# verify that "ga" and "gz" variables point to group begin/end
assert ga['type'] == 'begin_group'
assert gz['type'] == 'end_group'
assert ga.get('disabled') == gz.get('disabled') == None
# verify values before setting "disabled=TRUE"
fp = FormPack(scontent, id_string='xx')
section = next(iter(fp[0].sections.values()))
# only(?) way to access groups is in the hierarchy of a child question
n2_parent = section.fields['n2'].hierarchy[-2]
assert isinstance(n2_parent, FormGroup)
assert n2_parent.name == 'nada'
assert 'nada' in fp[0].to_xml()
ga['disabled'] = gz['disabled'] = 'TRUE'
fp = FormPack(scontent, id_string='xx')
section = next(iter(fp[0].sections.values()))
n2_parent = section.fields['n2'].hierarchy[-2]
# n2_parent is a "<FormSection name='submissions'>"
# test opposite of what's tested above--
assert not isinstance(n2_parent, FormGroup)
assert n2_parent.name != 'nada'
assert 'nada' not in fp[0].to_xml()
def test_list_fields_from_many_versions_on_packs(self):
title, schemas, submissions = build_fixture('site_inspection')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 5)
fields = {
field.name: field for field in fp.get_fields_for_versions(
fp.versions.keys())
}
field_names = sorted(fields.keys())
self.assertListEqual(field_names, [
'did_you_find_the_site',
'inspector',
'is_plant_life_encroaching',
'is_the_gate_secure',
'ping',
'please_rate_the_impact_of_any_defects_observed',
'rssi',
'was_there_damage_to_the_site',
'was_there_damage_to_the_site_dupe',
])
field_types = [fields[name].__class__.__name__ for name in field_names]
self.assertListEqual(field_types, [
'FormChoiceField',
'TextField',
'FormChoiceField',
'FormChoiceField',
'NumField',
'FormChoiceField',
'NumField',
'FormChoiceField',
'FormChoiceField',
])
def print_xls(filename, expand=False, flatten=False, xml=False):
"""
converts and XLS file with many sheets to a JSON object with lists
of key-value pairs for each row in the sheet.
"""
try:
with open(filename, 'r') as ff:
content = xls_to_dicts(ff)
if expand:
expand_content(content)
settings = content.get('settings', {})
settings['title'] = settings.get('title', 'title')
settings['id_string'] = settings.get('id_string', 'id_string')
content['settings'] = [settings]
if flatten:
flatten_content(content)
settings = content.pop('settings', [{}])[0]
if xml:
print(FormPack({'content': content}, **settings)[0].to_xml())
else:
print(json.dumps(content,
indent=2))
except EnvironmentError as e:
sys.exit('error trying to read input as xls file? {}: {}'.format(
filename, e))
def test_to_xml_fails_when_question_has_null_label():
# previously, this form would trigger a confusing error from pyxform:
# - Exception: (<type 'NoneType'>, None)
# it occurs when a named translation has a <NoneType> value
# (empty strings are OK)
fp = FormPack(
{
'content': {
'survey': [
{
'type': 'note',
'name': 'n1',
'label': [None],
},
],
'translations': ['NamedTranslation'],
'translated': ['label'],
}
},
id_string='sdf',
)
with pytest.raises(
pyxform.errors.PyXFormError,
match="^The survey element named 'n1' has no label or hint.$",
):
fp[0].to_xml()
def test_disaggregate_extended_fields(self):
title, schemas, submissions = build_fixture(
'auto_report_extended_fields')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, split_by="when")
assert stats.submissions_count == 22
stats = [(str(repr(field)), field_name, stats_dict)
for field, field_name, stats_dict in stats]
for stat in stats:
stats_dict = dict(stat[2])
for value in stats_dict.get("values"):
value_list = value[1]
percentage_responses = [
x[0] for x in value_list.get("percentage")
]
frequency_responses = [
x[0] for x in value_list.get("frequency")
]
assert percentage_responses == frequency_responses
assert percentage_responses[-1] == "..."
def setUp(self):
self.user = User.objects.get(username='******')
self.asset = Asset.objects.create(content=deepcopy(F1),
owner=self.user)
num_submissions = 4
submissions = []
for i in range(0, num_submissions):
submissions.append(
OrderedDict([(key, SUBMISSION_DATA[key][i])
for key in SUBMISSION_DATA.keys()]))
self.asset.deploy(backend='mock', active=True)
self.asset.save()
v_uid = self.asset.latest_deployed_version.uid
for submission in submissions:
submission.update({'__version__': v_uid})
self.asset.deployment._mock_submission(submission)
self.asset.save(create_version=False)
schemas = [
v.to_formpack_schema() for v in self.asset.deployed_versions
]
self.fp = FormPack(versions=schemas, id_string=self.asset.uid)
self.vs = self.fp.versions.keys()
self.submissions = self.asset.deployment._get_submissions()
def test_single_version_doesnt_require_version(self):
FormPack(
id_string='idstring',
versions=[
copy(SINGLE_NOTE_SURVEY),
],
)
def test_to_dict():
schema = build_fixture('restaurant_profile')[1][2]
_copy = deepcopy(schema)
fp = FormPack([schema], 'title')
original_content = _copy['content']
new_content = fp[0].to_dict()
assert original_content == new_content
def test_copy_fields_and_force_index_and_unicode(self):
title, schemas, submissions = customer_satisfaction
fp = FormPack(schemas, 'رضا العملاء')
export = fp.export(copy_fields=('_uuid', '_submission_time'),
force_index=True)
exported = export.to_dict(submissions)
expected = OrderedDict({
"رضا العملاء": {
'fields': [
"restaurant_name", "customer_enjoyment", "_uuid",
"_submission_time", "_index"
],
'data':
[[
"Felipes", "yes", "90dd7750f83011e590707c7a9125d07d",
"2016-04-01 19:57:45.306805", 1
],
[
"Dunkin Donuts", "no", "90dd7750f83011e590707c7a9125d08d",
"2016-04-02 19:57:45.306805", 2
],
[
"McDonalds", "no", "90dd7750f83011e590707c7a9125d09d",
"2016-04-03 19:57:45.306805", 3
]]
}
})
self.assertEqual(exported, expected)
with tempdir() as d:
xls = d / 'test.xlsx'
fp.export().to_xlsx(xls, submissions)
assert xls.isfile()
def build_formpack(id_string, xform):
schema = {
"id_string": id_string,
"version": 'v1',
"content": xform.to_kpi_content_schema(),
}
return xform, FormPack([schema], xform.title)
def test_site_inspection(self):
title, schemas, submissions = build_fixture('site_inspection')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 5)
v0 = fp[0]
self.assertEqual(list(v0.sections['Site inspection'].fields.keys()), [
u'inspector',
u'did_you_find_the_site',
u'was_there_damage_to_the_site',
u'was_there_damage_to_the_site_dupe',
u'ping',
u'rssi',
u'is_the_gate_secure',
u'is_plant_life_encroaching',
u'please_rate_the_impact_of_any_defects_observed',
])
self.assertEqual(sorted(fp.to_dict().keys()),
sorted([u'id_string', u'title', u'versions']))
self.assertEqual(
fp.to_dict(), {
u'title': u'Site inspection',
u'id_string': u'site_inspection',
u'versions': [s['content'] for s in schemas]
})
def test_copy_fields(self):
title, schemas, submissions = customer_satisfaction
forms = FormPack(schemas, title)
export = forms.export(copy_fields=('_uuid', '_submission_time'))
exported = export.to_dict(submissions)
expected = OrderedDict({
"Customer Satisfaction": {
'fields': [
"restaurant_name", "customer_enjoyment", "_uuid",
"_submission_time"
],
'data':
[[
"Felipes", "yes", "90dd7750f83011e590707c7a9125d07d",
"2016-04-01 19:57:45.306805"
],
[
"Dunkin Donuts", "no", "90dd7750f83011e590707c7a9125d08d",
"2016-04-02 19:57:45.306805"
],
[
"McDonalds", "no", "90dd7750f83011e590707c7a9125d09d",
"2016-04-03 19:57:45.306805"
]]
}
})
self.assertDictEquals(exported, expected)
def test_xlsx_sheet_name_limit(self):
'''
PyExcelerate will raise the following if any sheet name exceeds 31
characters:
Exception: Excel does not permit worksheet names longer than 31
characters. Set force_name=True to disable this restriction.
'''
title, schemas, submissions = build_fixture('long_names')
fp = FormPack(schemas, title)
options = {
'versions':
'long_survey_name__the_quick__brown_fox_jumps'
'_over_the_lazy_dog_v1'
}
with tempdir() as d:
xls = d / 'foo.xlsx'
fp.export(**options).to_xlsx(xls, submissions)
assert xls.isfile()
book = xlrd.open_workbook(xls)
assert book.sheet_names() == [
u'long survey name: the quick,...',
u'long_group_name__Victor_jagt...',
u'long_group_name__Victor_... (1)'
]
def test_to_xml():
"""
at the very least, version.to_xml() does not fail
"""
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
for version in fp.versions.keys():
fp.versions[version].to_xml()
def test_fixture_has_translations():
"""
restauraunt_profile@v2 has two translations
"""
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
assert len(fp[1].translations) == 2
def test_rich_report(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'frequency': [('Felipes', 2),
('The other one', 2),
('That one', 1)],
'not_provided': 1,
'percentage': [('Felipes', 33.33),
('The other one', 33.33),
('That one', 16.67)],
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<DateField name='when' type='date'>",
'when',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 2),
('2003-01-01', 1)],
'not_provided': 1,
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 33.33),
('2003-01-01', 16.67)],
'provided': 5,
'show_graph': True,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'mean': 1.6,
'median': 2,
'mode': 2,
'not_provided': 1,
'provided': 5,
'show_graph': False,
'stdev': 0.5477225575051661,
'total_count': 6}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_conflicting_version_ids(self):
with pytest.raises(ValueError):
FormPack(
id_string='idstring',
versions=[
copy(SINGLE_NOTE_SURVEY),
copy(SINGLE_NOTE_SURVEY),
],
)
def generate_xml_from_source(self,
source,
include_note=False,
root_node_name='snapshot_xml',
form_title=None,
id_string=None):
if form_title is None:
form_title = 'Snapshot XML'
if id_string is None:
id_string = 'snapshot_xml'
if include_note and 'survey' in source:
_translations = source.get('translations', [])
_label = include_note
if len(_translations) > 0:
_label = [_label for t in _translations]
source['survey'].append({
'type': 'note',
'name': 'prepended_note',
'label': _label
})
source_copy = copy.deepcopy(source)
self._expand_kobo_qs(source_copy)
self._populate_fields_with_autofields(source_copy)
self._strip_kuids(source_copy)
warnings = []
details = {}
try:
xml = FormPack({'content': source_copy},
root_node_name=root_node_name,
id_string=id_string,
title=form_title)[0].to_xml(warnings=warnings)
details.update({
'status': 'success',
'warnings': warnings,
})
except Exception as err:
err_message = str(err)
logging.error('Failed to generate xform for asset',
extra={
'src': source,
'id_string': id_string,
'uid': self.uid,
'_msg': err_message,
'warnings': warnings,
})
xml = ''
details.update({
'status': 'failure',
'error_type': type(err).__name__,
'error': err_message,
'warnings': warnings,
})
return xml, details
def test_repeats(self):
title, schemas, submissions = build_fixture('grouped_repeatable')
fp = FormPack(schemas, title)
options = {'versions': 'rgv1'}
export = fp.export(**options).to_dict(submissions)
self.assertEqual(
export,
OrderedDict([
('Household survey with repeatable groups', {
'fields': ['start', 'end', 'household_location', '_index'],
'data':
[[
'2016-03-14T14:15:48.000-04:00',
'2016-03-14T14:18:35.000-04:00', 'montreal', 1
],
[
'2016-03-14T14:14:10.000-04:00',
'2016-03-14T14:15:48.000-04:00', 'marseille', 2
],
[
'2016-03-14T14:13:53.000-04:00',
'2016-03-14T14:14:10.000-04:00', 'rocky mountains', 3
],
[
'2016-03-14T14:12:54.000-04:00',
'2016-03-14T14:13:53.000-04:00', 'toronto', 4
],
[
'2016-03-14T14:18:35.000-04:00',
'2016-03-14T15:19:20.000-04:00', 'new york', 5
],
[
'2016-03-14T14:11:25.000-04:00',
'2016-03-14T14:12:03.000-04:00', 'boston', 6
]]
}),
('houshold_member_repeat', {
'fields': [
'household_member_name', '_parent_table_name',
'_parent_index'
],
'data':
[['peter', 'Household survey with repeatable groups', 1],
['kyle', 'Household survey with repeatable groups', 2],
['linda', 'Household survey with repeatable groups', 2],
['morty', 'Household survey with repeatable groups', 3],
['tony', 'Household survey with repeatable groups', 4],
['mary', 'Household survey with repeatable groups', 4],
['emma', 'Household survey with repeatable groups', 5],
['parker', 'Household survey with repeatable groups', 5],
['amadou', 'Household survey with repeatable groups', 6],
['esteban', 'Household survey with repeatable groups', 6],
['suzie', 'Household survey with repeatable groups', 6],
['fiona', 'Household survey with repeatable groups', 6],
['phillip', 'Household survey with repeatable groups', 6]]
})
]))
def build_formpack(username, id_string):
user = User.objects.get(username=username)
xform = user.xforms.get(id_string=id_string)
schema = {
"id_string": id_string,
"version": 'v1',
"content": xform.to_kpi_content_schema(),
}
return user, xform, FormPack([schema], xform.title)
def test_xlsx(self):
title, schemas, submissions = build_fixture('grouped_repeatable')
fp = FormPack(schemas, title)
options = {'versions': 'rgv1'}
with tempdir() as d:
xls = d / 'foo.xlsx'
fp.export(**options).to_xlsx(xls, submissions)
assert xls.isfile()
def test_formpack_cannot_have_name(self):
with pytest.raises(TypeError):
vdata = copy(SINGLE_NOTE_SURVEY)
FormPack(
id_string='idstring',
name='somename',
versions=[
vdata,
],
)
def test_grouped_questions(self):
'''
questions groups
'''
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 1)
self.assertEqual(
list(fp[0].sections['Grouped questions'].fields.keys()),
['q1', 'g1q1', 'g1sg1q1', 'g1q2', 'g2q1', 'qz'])
def test_headers_of_group_exports(self):
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
options = {'versions': 'gqs'}
# by default, groups are stripped.
export = fp.export(**options).to_dict(submissions)
headers = export['Grouped questions']['fields']
self.assertEquals(headers,
['q1', 'g1q1', 'g1sg1q1', 'g1q2', 'g2q1', 'qz'])
def test_restaurant_profile(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 4)
v0 = fp[0]
self.assertEqual(list(v0.sections['Restaurant profile'].fields.keys()),
[u'restaurant_name', u'location'])
self.assertEqual(sorted(fp.to_dict().keys()),
sorted([u'id_string', u'title', u'versions']))
def test_formpack_version_cannot_have_name(self):
with pytest.raises(ValueError):
vdata = copy(SINGLE_NOTE_SURVEY)
vdata['name'] = 'somename'
FormPack(
id_string='idstring',
versions=[
vdata,
],
)
def test_get_fields_for_versions_returns_unique_fields():
"""
As described in #127, `get_field_for_versions()` would return identical
fields multiple times. This is was a failing test to reproduce that issue
"""
fp = FormPack([
{
'content': {
'survey': [
{
'name': 'hey',
'type': 'image'
},
{
'name': 'two',
'type': 'image'
},
]
},
'version': 'vRR7hH6SxTupvtvCqu7n5d',
},
{
'content': {
'survey': [
{
'name': 'one',
'type': 'image'
},
{
'name': 'two',
'type': 'image'
},
]
},
'version': 'vA8xs9JVi8aiSfypLgyYW2',
},
{
'content': {
'survey': [
{
'name': 'one',
'type': 'image'
},
{
'name': 'two',
'type': 'image'
},
]
},
'version': 'vNqgh8fJqyjFk6jgiCk4rn',
},
])
fields = fp.get_fields_for_versions(fp.versions)
field_names = [field.name for field in fields]
assert sorted(field_names) == ['hey', 'one', 'two']
def test_sanitation_report(self):
'''
sanitation_report
'''
title, schemas, submissions = build_fixture('sanitation_report')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 1)
v0 = fp[0]
self.assertEqual(
list(v0.sections['Sanitation report'].fields.keys()),
['restaurant_name', 'restaurant_rating', 'report_date'])
def test_simple_report(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, lang='french')
assert stats.submissions_count == 4
stats = [(str(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'nom du restaurant', {
'frequency': [('Taco Truck', 1), ('Harvest', 1),
('Wololo', 1), ('Los pollos hermanos', 1)],
'not_provided':
0,
'percentage': [('Taco Truck', 25.00), ('Harvest', 25.00),
('Wololo', 25.00),
('Los pollos hermanos', 25.00)],
'provided':
4,
'show_graph':
False,
'total_count':
4
}),
("<FormGPSField name='location' type='geopoint'>", 'lieu', {
'not_provided': 0,
'provided': 4,
'show_graph': False,
'total_count': 4
}),
("<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>",
'type de restaurant', {
'frequency': [('traditionnel', 2),
('avec vente \xe0 emporter', 1)],
'not_provided':
1,
'percentage': [('traditionnel', 50.00),
('avec vente \xe0 emporter', 25.00)],
'provided':
3,
'show_graph':
True,
'total_count':
4
})
]
for i, stat in enumerate(stats):
assert stat == expected[i]
