def test_copy_fields(self):
title, schemas, submissions = customer_satisfaction
forms = FormPack(schemas, title)
export = forms.export(copy_fields=('_uuid', '_submission_time'))
exported = export.to_dict(submissions)
expected = OrderedDict({
"Customer Satisfaction": {
'fields': [
"restaurant_name", "customer_enjoyment", "_uuid",
"_submission_time"
],
'data':
[[
"Felipes", "yes", "90dd7750f83011e590707c7a9125d07d",
"2016-04-01 19:57:45.306805"
],
[
"Dunkin Donuts", "no", "90dd7750f83011e590707c7a9125d08d",
"2016-04-02 19:57:45.306805"
],
[
"McDonalds", "no", "90dd7750f83011e590707c7a9125d09d",
"2016-04-03 19:57:45.306805"
]]
}
})
self.assertDictEquals(exported, expected)
def test_copy_fields_and_force_index_and_unicode(self):
title, schemas, submissions = customer_satisfaction
fp = FormPack(schemas, 'رضا العملاء')
export = fp.export(copy_fields=('_uuid', '_submission_time'),
force_index=True)
exported = export.to_dict(submissions)
expected = OrderedDict({
"رضا العملاء": {
'fields': [
"restaurant_name", "customer_enjoyment", "_uuid",
"_submission_time", "_index"
],
'data':
[[
"Felipes", "yes", "90dd7750f83011e590707c7a9125d07d",
"2016-04-01 19:57:45.306805", 1
],
[
"Dunkin Donuts", "no", "90dd7750f83011e590707c7a9125d08d",
"2016-04-02 19:57:45.306805", 2
],
[
"McDonalds", "no", "90dd7750f83011e590707c7a9125d09d",
"2016-04-03 19:57:45.306805", 3
]]
}
})
self.assertEqual(exported, expected)
with tempdir() as d:
xls = d / 'test.xlsx'
fp.export().to_xlsx(xls, submissions)
assert xls.isfile()
def test_site_inspection(self):
title, schemas, submissions = build_fixture('site_inspection')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 5)
v0 = fp[0]
self.assertEqual(list(v0.sections['Site inspection'].fields.keys()), [
u'inspector',
u'did_you_find_the_site',
u'was_there_damage_to_the_site',
u'was_there_damage_to_the_site_dupe',
u'ping',
u'rssi',
u'is_the_gate_secure',
u'is_plant_life_encroaching',
u'please_rate_the_impact_of_any_defects_observed',
])
self.assertEqual(sorted(fp.to_dict().keys()),
sorted([u'id_string', u'title', u'versions']))
self.assertEqual(
fp.to_dict(), {
u'title': u'Site inspection',
u'id_string': u'site_inspection',
u'versions': [s['content'] for s in schemas]
})
def test_disaggregate_extended_fields(self):
title, schemas, submissions = build_fixture(
'auto_report_extended_fields')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, split_by="when")
assert stats.submissions_count == 22
stats = [(str(repr(field)), field_name, stats_dict)
for field, field_name, stats_dict in stats]
for stat in stats:
stats_dict = dict(stat[2])
for value in stats_dict.get("values"):
value_list = value[1]
percentage_responses = [
x[0] for x in value_list.get("percentage")
]
frequency_responses = [
x[0] for x in value_list.get("frequency")
]
assert percentage_responses == frequency_responses
assert percentage_responses[-1] == "..."
def test_xlsx_sheet_name_limit(self):
'''
PyExcelerate will raise the following if any sheet name exceeds 31
characters:
Exception: Excel does not permit worksheet names longer than 31
characters. Set force_name=True to disable this restriction.
'''
title, schemas, submissions = build_fixture('long_names')
fp = FormPack(schemas, title)
options = {
'versions':
'long_survey_name__the_quick__brown_fox_jumps'
'_over_the_lazy_dog_v1'
}
with tempdir() as d:
xls = d / 'foo.xlsx'
fp.export(**options).to_xlsx(xls, submissions)
assert xls.isfile()
book = xlrd.open_workbook(xls)
assert book.sheet_names() == [
u'long survey name: the quick,...',
u'long_group_name__Victor_jagt...',
u'long_group_name__Victor_... (1)'
]
def test_list_fields_from_many_versions_on_packs(self):
title, schemas, submissions = build_fixture('site_inspection')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 5)
fields = {
field.name: field for field in fp.get_fields_for_versions(
fp.versions.keys())
}
field_names = sorted(fields.keys())
self.assertListEqual(field_names, [
'did_you_find_the_site',
'inspector',
'is_plant_life_encroaching',
'is_the_gate_secure',
'ping',
'please_rate_the_impact_of_any_defects_observed',
'rssi',
'was_there_damage_to_the_site',
'was_there_damage_to_the_site_dupe',
])
field_types = [fields[name].__class__.__name__ for name in field_names]
self.assertListEqual(field_types, [
'FormChoiceField',
'TextField',
'FormChoiceField',
'FormChoiceField',
'NumField',
'FormChoiceField',
'NumField',
'FormChoiceField',
'FormChoiceField',
])
def test_rich_report(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'frequency': [('Felipes', 2),
('The other one', 2),
('That one', 1)],
'not_provided': 1,
'percentage': [('Felipes', 33.33),
('The other one', 33.33),
('That one', 16.67)],
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<DateField name='when' type='date'>",
'when',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 2),
('2003-01-01', 1)],
'not_provided': 1,
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 33.33),
('2003-01-01', 16.67)],
'provided': 5,
'show_graph': True,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'mean': 1.6,
'median': 2,
'mode': 2,
'not_provided': 1,
'provided': 5,
'show_graph': False,
'stdev': 0.5477225575051661,
'total_count': 6}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_rich_report(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'frequency': [('Felipes', 2),
('The other one', 2),
('That one', 1)],
'not_provided': 1,
'percentage': [('Felipes', 33.33),
('The other one', 33.33),
('That one', 16.67)],
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<DateField name='when' type='date'>",
'when',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 2),
('2003-01-01', 1)],
'not_provided': 1,
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 33.33),
('2003-01-01', 16.67)],
'provided': 5,
'show_graph': True,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'mean': 1.6,
'median': 2,
'mode': 2,
'not_provided': 1,
'provided': 5,
'show_graph': False,
'stdev': 0.5477225575051661,
'total_count': 6}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_xlsx(self):
title, schemas, submissions = build_fixture('grouped_repeatable')
fp = FormPack(schemas, title)
options = {'versions': 'rgv1'}
with tempdir() as d:
xls = d / 'foo.xlsx'
fp.export(**options).to_xlsx(xls, submissions)
assert xls.isfile()
def test_repeats(self):
title, schemas, submissions = build_fixture('grouped_repeatable')
fp = FormPack(schemas, title)
options = {'versions': 'rgv1'}
export = fp.export(**options).to_dict(submissions)
self.assertEqual(
export,
OrderedDict([
('Household survey with repeatable groups', {
'fields': ['start', 'end', 'household_location', '_index'],
'data':
[[
'2016-03-14T14:15:48.000-04:00',
'2016-03-14T14:18:35.000-04:00', 'montreal', 1
],
[
'2016-03-14T14:14:10.000-04:00',
'2016-03-14T14:15:48.000-04:00', 'marseille', 2
],
[
'2016-03-14T14:13:53.000-04:00',
'2016-03-14T14:14:10.000-04:00', 'rocky mountains', 3
],
[
'2016-03-14T14:12:54.000-04:00',
'2016-03-14T14:13:53.000-04:00', 'toronto', 4
],
[
'2016-03-14T14:18:35.000-04:00',
'2016-03-14T15:19:20.000-04:00', 'new york', 5
],
[
'2016-03-14T14:11:25.000-04:00',
'2016-03-14T14:12:03.000-04:00', 'boston', 6
]]
}),
('houshold_member_repeat', {
'fields': [
'household_member_name', '_parent_table_name',
'_parent_index'
],
'data':
[['peter', 'Household survey with repeatable groups', 1],
['kyle', 'Household survey with repeatable groups', 2],
['linda', 'Household survey with repeatable groups', 2],
['morty', 'Household survey with repeatable groups', 3],
['tony', 'Household survey with repeatable groups', 4],
['mary', 'Household survey with repeatable groups', 4],
['emma', 'Household survey with repeatable groups', 5],
['parker', 'Household survey with repeatable groups', 5],
['amadou', 'Household survey with repeatable groups', 6],
['esteban', 'Household survey with repeatable groups', 6],
['suzie', 'Household survey with repeatable groups', 6],
['fiona', 'Household survey with repeatable groups', 6],
['phillip', 'Household survey with repeatable groups', 6]]
})
]))
def test_xlsx(self):
title, schemas, submissions = build_fixture('grouped_repeatable')
fp = FormPack(schemas, title)
options = {'versions': 'rgv1'}
with tempdir() as d:
xls = d / 'foo.xlsx'
fp.export(**options).to_xlsx(xls, submissions)
assert xls.isfile()
def test_headers_of_group_exports(self):
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
options = {'versions': 'gqs'}
# by default, groups are stripped.
export = fp.export(**options).to_dict(submissions)
headers = export['Grouped questions']['fields']
self.assertEquals(headers,
['q1', 'g1q1', 'g1sg1q1', 'g1q2', 'g2q1', 'qz'])
def test_restaurant_profile(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 4)
v0 = fp[0]
self.assertEqual(list(v0.sections['Restaurant profile'].fields.keys()),
[u'restaurant_name', u'location'])
self.assertEqual(sorted(fp.to_dict().keys()),
sorted([u'id_string', u'title', u'versions']))
def test_headers_of_group_exports(self):
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
options = {'versions': 'gqs'}
# by default, groups are stripped.
export = fp.export(**options).to_dict(submissions)
headers = export['Grouped questions']['fields']
self.assertEquals(headers, ['q1', 'g1q1', 'g1sg1q1',
'g1q2', 'g2q1', 'qz'])
def test_restaurant_profile(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 4)
v0 = fp[0]
self.assertEqual(list(v0.sections['Restaurant profile'].fields.keys()),
[u'restaurant_name', u'location'])
self.assertEqual(sorted(fp.to_dict().keys()),
sorted([u'id_string', u'title', u'versions']))
def test_get_fields_for_versions_returns_unique_fields():
"""
As described in #127, `get_field_for_versions()` would return identical
fields multiple times. This is was a failing test to reproduce that issue
"""
fp = FormPack([
{
'content': {
'survey': [
{
'name': 'hey',
'type': 'image'
},
{
'name': 'two',
'type': 'image'
},
]
},
'version': 'vRR7hH6SxTupvtvCqu7n5d',
},
{
'content': {
'survey': [
{
'name': 'one',
'type': 'image'
},
{
'name': 'two',
'type': 'image'
},
]
},
'version': 'vA8xs9JVi8aiSfypLgyYW2',
},
{
'content': {
'survey': [
{
'name': 'one',
'type': 'image'
},
{
'name': 'two',
'type': 'image'
},
]
},
'version': 'vNqgh8fJqyjFk6jgiCk4rn',
},
])
fields = fp.get_fields_for_versions(fp.versions)
field_names = [field.name for field in fields]
assert sorted(field_names) == ['hey', 'one', 'two']
def test_stats_with_non_numeric_value_for_numeric_field(self):
"""
A string response to an integer question, for example, should not cause
a crash; it should be treated as if no response was provided
"""
title = 'Just one number'
schemas = [{
'content': {
'survey': [{
'type': 'integer',
'name': 'the_number',
'label': 'Enter the number!',
}]
}
}]
submissions = [
{
'the_number': 10
},
{
'the_number': 20
},
{
'the_number': 30
},
{
'the_number': 'oops!'
},
]
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == len(submissions)
stats = [(str(repr(f)), n, d) for f, n, d in stats]
expected = [(
"<NumField name='the_number' type='integer'>",
'the_number',
{
'mean': 20.0,
'median': 20,
'mode': '*',
'not_provided': 1,
'provided': 3,
'show_graph': False,
'stdev': 10.0,
'total_count': 4,
},
)]
for i, stat in enumerate(stats):
assert stat == expected[i]
def test_simple_report(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, lang='french')
assert stats.submissions_count == 4
stats = [(str(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'nom du restaurant', {
'frequency': [('Taco Truck', 1), ('Harvest', 1),
('Wololo', 1), ('Los pollos hermanos', 1)],
'not_provided':
0,
'percentage': [('Taco Truck', 25.00), ('Harvest', 25.00),
('Wololo', 25.00),
('Los pollos hermanos', 25.00)],
'provided':
4,
'show_graph':
False,
'total_count':
4
}),
("<FormGPSField name='location' type='geopoint'>", 'lieu', {
'not_provided': 0,
'provided': 4,
'show_graph': False,
'total_count': 4
}),
("<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>",
'type de restaurant', {
'frequency': [('traditionnel', 2),
('avec vente \xe0 emporter', 1)],
'not_provided':
1,
'percentage': [('traditionnel', 50.00),
('avec vente \xe0 emporter', 25.00)],
'provided':
3,
'show_graph':
True,
'total_count':
4
})
]
for i, stat in enumerate(stats):
assert stat == expected[i]
def test_customer_satisfaction(self):
'''
customer_satisfaction
'''
title, schemas, submissions = build_fixture('customer_satisfaction')
fp = FormPack(schemas, title)
v0 = fp[0]
self.assertEqual(len(fp.versions), 1)
self.assertEqual(list(v0.sections['Customer Satisfaction'].fields.keys()),
[u'restaurant_name', u'customer_enjoyment'])
self.assertEqual(sorted(fp.to_dict().keys()),
[u'id_string', u'title', u'versions'])
def test_list_fields_on_packs(self):
title, schemas, _ = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
fields = fp.get_fields_for_versions()
field_names = [field.name for field in fields]
assert field_names == ['restaurant_name', 'location', 'eatery_type']
field_types = [field.__class__.__name__ for field in fields]
assert ' '.join(field_types) == ' '.join(['TextField', 'FormGPSField',
'FormChoiceFieldWithMultipleSelect'])
def test_customer_satisfaction(self):
'''
customer_satisfaction
'''
title, schemas, submissions = build_fixture('customer_satisfaction')
fp = FormPack(schemas, title)
v0 = fp[0]
self.assertEqual(len(fp.versions), 1)
self.assertEqual(
list(v0.sections['Customer Satisfaction'].fields.keys()),
[u'restaurant_name', u'customer_enjoyment'])
self.assertEqual(sorted(fp.to_dict().keys()),
[u'id_string', u'title', u'versions'])
def test_fields_for_versions_list_index_out_of_range():
title, schemas, submissions = build_fixture(
'fields_for_versions_list_index_out_of_range')
fp = FormPack(schemas, title)
all_fields = fp.get_fields_for_versions(fp.versions.keys())
expected = [
'one',
'third',
'first_but_not_one',
]
field_names = [field.name for field in all_fields]
assert len(all_fields) == 3
assert field_names == expected
def test_submission_counts_match():
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport(versions=fp.versions.keys())
stats = report.get_stats(submissions)
assert stats.submissions_count == len(submissions)
assert stats.submission_counts_by_version == {
u'rpv1': 1,
u'rpV2': 1,
u'rpV3': 2,
u'rpV4': 4,
}
def test_list_fields_on_packs(self):
title, schemas, _ = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
fields = fp.get_fields_for_versions()
field_names = [field.name for field in fields]
assert field_names == ['restaurant_name', 'location', 'eatery_type']
field_types = [field.__class__.__name__ for field in fields]
assert ' '.join(field_types) == ' '.join(['TextField', 'FormGPSField',
'FormChoiceFieldWithMultipleSelect'])
def test_groups_disabled():
scontent = {
'content': {
'survey': [
{
'type': 'text',
'name': 'n1',
'label': ['aa']
},
{
'type': 'begin_group',
'name': 'nada'
},
{
'type': 'note',
'name': 'n2',
'label': ['ab']
},
{
'type': 'end_group'
},
],
'translations': ['en'],
'translated': ['label'],
}
}
(ga, gz) = [scontent['content']['survey'][nn] for nn in [1, 3]]
# verify that "ga" and "gz" variables point to group begin/end
assert ga['type'] == 'begin_group'
assert gz['type'] == 'end_group'
assert ga.get('disabled') == gz.get('disabled') == None
# verify values before setting "disabled=TRUE"
fp = FormPack(scontent, id_string='xx')
section = next(iter(fp[0].sections.values()))
# only(?) way to access groups is in the hierarchy of a child question
n2_parent = section.fields['n2'].hierarchy[-2]
assert isinstance(n2_parent, FormGroup)
assert n2_parent.name == 'nada'
assert 'nada' in fp[0].to_xml()
ga['disabled'] = gz['disabled'] = 'TRUE'
fp = FormPack(scontent, id_string='xx')
section = next(iter(fp[0].sections.values()))
n2_parent = section.fields['n2'].hierarchy[-2]
# n2_parent is a "<FormSection name='submissions'>"
# test opposite of what's tested above--
assert not isinstance(n2_parent, FormGroup)
assert n2_parent.name != 'nada'
assert 'nada' not in fp[0].to_xml()
def test_simple_report(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, lang='french')
assert stats.submissions_count == 4
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
(
"<TextField name='restaurant_name' type='text'>",
'nom du restaurant',
{'frequency': [('Taco Truck', 1),
('Harvest', 1),
('Los pollos hermanos', 1),
('Wololo', 1)],
'not_provided': 0,
'percentage': [('Taco Truck', 25.00),
('Harvest', 25.00),
('Los pollos hermanos', 25.00),
('Wololo', 25.00)],
'provided': 4,
'show_graph': False,
'total_count': 4}
),
(
"<FormGPSField name='location' type='geopoint'>",
'lieu',
{'not_provided': 0,
'provided': 4,
'show_graph': False,
'total_count': 4}
),
(
"<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>",
'type de restaurant',
{'frequency': [('traditionnel', 2), ('avec vente \xe0 emporter', 1)],
'not_provided': 1,
'percentage': [('traditionnel', 50.00),
('avec vente \xe0 emporter', 25.00)],
'provided': 3,
'show_graph': True,
'total_count': 4}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_force_index(self):
title, schemas, submissions = customer_satisfaction
forms = FormPack(schemas, title)
export = forms.export(force_index=True).to_dict(submissions)
expected = OrderedDict({
"Customer Satisfaction": {
'fields': ["restaurant_name", "customer_enjoyment", "_index"],
'data': [["Felipes", "yes", 1], ["Dunkin Donuts", "no", 2],
["McDonalds", "no", 3]]
}
})
self.assertEqual(export, expected)
def test_generator_export_translation_headers(self):
title, schemas, submissions = restaurant_profile
fp = FormPack(schemas, title)
self.assertEqual(len(fp.versions), 4)
self.assertEqual(len(fp[1].translations), 2)
# by default, exports use the question 'name' attribute
headers = fp.export(versions=0).to_dict(submissions)['Restaurant profile']['fields']
self.assertEquals(headers, ['restaurant_name',
'location',
'_location_latitude',
'_location_longitude',
'_location_altitude',
'_location_precision'])
# the first translation in the list is the translation that
# appears first in the column list. in this case, 'label::english'
translations = fp[1].translations
export = fp.export(lang=translations[0], versions=1)
data = export.to_dict(submissions)
headers = data['Restaurant profile']['fields']
self.assertEquals(headers, ['restaurant name',
'location',
'_location_latitude',
'_location_longitude',
'_location_altitude',
'_location_precision'])
export = fp.export(lang=translations[1], versions=1)
data = export.to_dict(submissions)
headers = data['Restaurant profile']['fields']
self.assertEquals(headers, ['nom du restaurant',
'lieu',
'_lieu_latitude',
'_lieu_longitude',
'_lieu_altitude',
'_lieu_precision'])
# TODO: make a separate test to test to test __getitem__
export = fp.export(lang=UNTRANSLATED, versions='rpv1')
data = export.to_dict(submissions)
headers = data['Restaurant profile']['fields']
self.assertEquals(headers, ['restaurant name',
'location',
'_location_latitude',
'_location_longitude',
'_location_altitude',
'_location_precision'])
def print_xls(filename, expand=False, flatten=False, xml=False):
"""
converts and XLS file with many sheets to a JSON object with lists
of key-value pairs for each row in the sheet.
"""
try:
with open(filename, 'r') as ff:
content = xls_to_dicts(ff)
if expand:
expand_content(content)
settings = content.get('settings', {})
settings['title'] = settings.get('title', 'title')
settings['id_string'] = settings.get('id_string', 'id_string')
content['settings'] = [settings]
if flatten:
flatten_content(content)
settings = content.pop('settings', [{}])[0]
if xml:
print(FormPack({'content': content}, **settings)[0].to_xml())
else:
print(json.dumps(content,
indent=2))
except EnvironmentError as e:
sys.exit('error trying to read input as xls file? {}: {}'.format(
filename, e))
def test_to_xml_fails_when_question_has_null_label():
# previously, this form would trigger a confusing error from pyxform:
# - Exception: (<type 'NoneType'>, None)
# it occurs when a named translation has a <NoneType> value
# (empty strings are OK)
fp = FormPack(
{
'content': {
'survey': [
{
'type': 'note',
'name': 'n1',
'label': [None],
},
],
'translations': ['NamedTranslation'],
'translated': ['label'],
}
},
id_string='sdf',
)
with pytest.raises(
pyxform.errors.PyXFormError,
match="^The survey element named 'n1' has no label or hint.$",
):
fp[0].to_xml()
def setUp(self):
self.user = User.objects.get(username='******')
self.asset = Asset.objects.create(content=deepcopy(F1),
owner=self.user)
num_submissions = 4
submissions = []
for i in range(0, num_submissions):
submissions.append(
OrderedDict([(key, SUBMISSION_DATA[key][i])
for key in SUBMISSION_DATA.keys()]))
self.asset.deploy(backend='mock', active=True)
self.asset.save()
v_uid = self.asset.latest_deployed_version.uid
for submission in submissions:
submission.update({'__version__': v_uid})
self.asset.deployment._mock_submission(submission)
self.asset.save(create_version=False)
schemas = [
v.to_formpack_schema() for v in self.asset.deployed_versions
]
self.fp = FormPack(versions=schemas, id_string=self.asset.uid)
self.vs = self.fp.versions.keys()
self.submissions = self.asset.deployment._get_submissions()
def test_headers_of_multi_version_exports(self):
title, schemas, submissions = build_fixture('site_inspection')
fp = FormPack(schemas, title)
export = fp.export(versions=fp.versions.keys()).to_dict(submissions)
headers = export['Site inspection']['fields']
self.assertListEqual(headers, [
'inspector',
'did_you_find_the_site',
'was_there_damage_to_the_site',
'was_there_damage_to_the_site_dupe',
'ping',
'rssi',
'is_the_gate_secure',
'is_plant_life_encroaching',
'please_rate_the_impact_of_any_defects_observed',
])
def test_to_dict():
schema = build_fixture('restaurant_profile')[1][2]
_copy = deepcopy(schema)
fp = FormPack([schema], 'title')
original_content = _copy['content']
new_content = fp[0].to_dict()
assert original_content == new_content
def build_formpack(id_string, xform):
schema = {
"id_string": id_string,
"version": 'v1',
"content": xform.to_kpi_content_schema(),
}
return xform, FormPack([schema], xform.title)
def test_single_version_doesnt_require_version(self):
FormPack(
id_string='idstring',
versions=[
copy(SINGLE_NOTE_SURVEY),
],
)
def test_get_fields_for_versions_returns_newest_of_fields_with_same_name():
schemas = [
{
'version': 'v1',
'content': {
'survey': [
{
'name': 'constant_question_name',
'type': 'select_one choice',
'label': 'first version question label',
'hxl': '#first_version_hxl',
},
],
'choices': [
{
'list_name': 'choice',
'name': 'constant_choice_name',
'label': 'first version choice label',
},
],
},
},
{
'version': 'v2',
'content': {
'survey': [
{
'name': 'constant_question_name',
'type': 'select_one choice',
'label': 'second version question label',
'hxl': '#second_version_hxl',
},
],
'choices': [{
'list_name': 'choice',
'name': 'constant_choice_name',
'label': 'second version choice label',
}],
},
},
]
fp = FormPack(schemas)
fields = fp.get_fields_for_versions(fp.versions)
# The first and only field returned should be the first field of the first
# section of the last version
section_value = get_first_occurrence(fp[-1].sections.values())
assert fields[0] == get_first_occurrence(section_value.fields.values())
def test_to_xml():
"""
at the very least, version.to_xml() does not fail
"""
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
for version in fp.versions.keys():
fp.versions[version].to_xml()
def test_field_position_with_multiple_versions():
title, schemas, submissions = build_fixture(
'field_position_with_multiple_versions')
fp = FormPack(schemas, title)
all_fields = fp.get_fields_for_versions(fp.versions.keys())
expected = [
'City',
'Firstname',
'Lastname',
'Gender',
'Age',
'Fullname',
]
field_names = [field.name for field in all_fields]
assert len(all_fields) == 6
assert field_names == expected
def test_copy_fields_and_force_index_and_unicode(self):
title, schemas, submissions = customer_satisfaction
fp = FormPack(schemas, 'رضا العملاء')
export = fp.export(copy_fields=('_uuid', '_submission_time'),
force_index=True)
exported = export.to_dict(submissions)
expected = OrderedDict({
"رضا العملاء": {
'fields': ["restaurant_name", "customer_enjoyment",
"_uuid", "_submission_time", "_index"],
'data': [
[
"Felipes",
"yes",
"90dd7750f83011e590707c7a9125d07d",
"2016-04-01 19:57:45.306805",
1
],
[
"Dunkin Donuts",
"no",
"90dd7750f83011e590707c7a9125d08d",
"2016-04-02 19:57:45.306805",
2
],
[
"McDonalds",
"no",
"90dd7750f83011e590707c7a9125d09d",
"2016-04-03 19:57:45.306805",
3
]
]
}
})
self.assertEqual(exported, expected)
with tempdir() as d:
xls = d / 'test.xlsx'
fp.export().to_xlsx(xls, submissions)
assert xls.isfile()
def test_force_index(self):
title, schemas, submissions = customer_satisfaction
forms = FormPack(schemas, title)
export = forms.export(force_index=True).to_dict(submissions)
expected = OrderedDict({
"Customer Satisfaction": {
'fields': ["restaurant_name", "customer_enjoyment",
"_index"],
'data': [
["Felipes", "yes", 1],
["Dunkin Donuts", "no", 2],
["McDonalds", "no", 3]
]
}
})
self.assertEqual(export, expected)
def test_choices_external_as_text_field(self):
title, schemas, submissions = build_fixture('sanitation_report_external')
fp = FormPack(schemas, title)
export = fp.export(lang=UNTRANSLATED)
exported = export.to_dict(submissions)
expected = OrderedDict([
(
'Sanitation report external', {
'fields': [
'Restaurant name',
'How did this restaurant do on its sanitation report?',
'Report date'
],
'data': [
[
'Felipes',
'A',
'012345'
],
[
'Chipotle',
'C',
'012346'
],
[
'Dunkin Donuts',
'D',
'012347'
],
[
'Boloco',
'B',
'012348'
]
]
}
)
])
self.assertEqual(exported, expected)
def test_submissions_of_group_exports(self):
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
options = {'versions': 'gqs'}
export = fp.export(**options).to_dict(submissions)['Grouped questions']
self.assertDictEquals(export['fields'], ['q1', 'g1q1', 'g1sg1q1',
'g1q2', 'g2q1', 'qz'])
self.assertDictEquals(export['data'], [['respondent1\'s r1',
'respondent1\'s r2',
'respondent1\'s r2.5',
'respondent1\'s r2.75 :)',
'respondent1\'s r3',
'respondent1\'s r4'],
['respondent2\'s r1',
'respondent2\'s r2',
'respondent2\'s r2.5',
'respondent2\'s r2.75 :)',
'respondent2\'s r3',
'respondent2\'s r4']])
options['hierarchy_in_labels'] = '/'
export = fp.export(**options).to_dict(submissions)['Grouped questions']
self.assertDictEquals(export['fields'], ['q1',
'g1/g1q1',
'g1/sg1/g1sg1q1',
'g1/g1q2',
'g2/g2q1',
'qz'])
self.assertDictEquals(export['data'], [['respondent1\'s r1',
'respondent1\'s r2',
'respondent1\'s r2.5',
'respondent1\'s r2.75 :)',
'respondent1\'s r3',
'respondent1\'s r4'],
['respondent2\'s r1',
'respondent2\'s r2',
'respondent2\'s r2.5',
'respondent2\'s r2.75 :)',
'respondent2\'s r3',
'respondent2\'s r4']])
def test_copy_fields(self):
title, schemas, submissions = customer_satisfaction
forms = FormPack(schemas, title)
export = forms.export(copy_fields=('_uuid', '_submission_time'))
exported = export.to_dict(submissions)
expected = OrderedDict({
"Customer Satisfaction": {
'fields': ["restaurant_name", "customer_enjoyment",
"_uuid", "_submission_time"],
'data': [
[
"Felipes",
"yes",
"90dd7750f83011e590707c7a9125d07d",
"2016-04-01 19:57:45.306805"
],
[
"Dunkin Donuts",
"no",
"90dd7750f83011e590707c7a9125d08d",
"2016-04-02 19:57:45.306805"
],
[
"McDonalds",
"no",
"90dd7750f83011e590707c7a9125d09d",
"2016-04-03 19:57:45.306805"
]
]
}
})
self.assertDictEquals(exported, expected)
def test_csv(self):
title, schemas, submissions = build_fixture('grouped_questions')
fp = FormPack(schemas, title)
options = {'versions': 'gqs'}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"q1";"g1q1";"g1sg1q1";"g1q2";"g2q1";"qz"
"respondent1's r1";"respondent1's r2";"respondent1's r2.5";"respondent1's r2.75 :)";"respondent1's r3";"respondent1's r4"
"respondent2's r1";"respondent2's r2";"respondent2's r2.5";"respondent2's r2.75 :)";"respondent2's r3";"respondent2's r4"
"""
self.assertTextEqual(csv_data, expected)
options = {'versions': 'gqs', 'hierarchy_in_labels': True}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"q1";"g1/g1q1";"g1/sg1/g1sg1q1";"g1/g1q2";"g2/g2q1";"qz"
"respondent1's r1";"respondent1's r2";"respondent1's r2.5";"respondent1's r2.75 :)";"respondent1's r3";"respondent1's r4"
"respondent2's r1";"respondent2's r2";"respondent2's r2.5";"respondent2's r2.75 :)";"respondent2's r3";"respondent2's r4"
"""
self.assertTextEqual(csv_data, expected)
options = {'versions': 'gqs', 'hierarchy_in_labels': True,
'lang': UNTRANSLATED}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"Q1";"Group 1/G1Q1";"Group 1/Sub Group 1/G1SG1Q1";"Group 1/G1Q2";"g2/G2Q1";"QZed"
"respondent1's r1";"respondent1's r2";"respondent1's r2.5";"respondent1's r2.75 :)";"respondent1's r3";"respondent1's r4"
"respondent2's r1";"respondent2's r2";"respondent2's r2.5";"respondent2's r2.75 :)";"respondent2's r3";"respondent2's r4"
"""
self.assertTextEqual(csv_data, expected)
title, schemas, submissions = restaurant_profile
fp = FormPack(schemas, title)
options = {'versions': 'rpV3', 'lang': fp[1].translations[1]}
csv_data = "\n".join(fp.export(**options).to_csv(submissions))
expected = """
"nom du restaurant";"lieu";"_lieu_latitude";"_lieu_longitude";"_lieu_altitude";"_lieu_precision";"type de restaurant"
"Taco Truck";"13.42 -25.43";"13.42";"-25.43";"";"";"avec vente à emporter"
"Harvest";"12.43 -24.53";"12.43";"-24.53";"";"";"traditionnel"
"""
self.assertTextEqual(csv_data, expected)
def test_repeats_alias(self):
title, schemas, submissions = build_fixture('grouped_repeatable_alias')
fp = FormPack(schemas, title)
options = {'versions': 'rgv1'}
export = fp.export(**options).to_dict(submissions)
self.assertEqual(export, OrderedDict ([
('Grouped Repeatable Alias',
{
'fields': [
'start',
'end',
'household_location',
'_index'
],
'data': [
[
'2016-03-14T14:15:48.000-04:00',
'2016-03-14T14:18:35.000-04:00',
'montreal',
1
],
[
'2016-03-14T14:14:10.000-04:00',
'2016-03-14T14:15:48.000-04:00',
'marseille',
2
],
[
'2016-03-14T14:13:53.000-04:00',
'2016-03-14T14:14:10.000-04:00',
'rocky mountains',
3
],
[
'2016-03-14T14:12:54.000-04:00',
'2016-03-14T14:13:53.000-04:00',
'toronto',
4
],
[
'2016-03-14T14:18:35.000-04:00',
'2016-03-14T15:19:20.000-04:00',
'new york',
5
],
[
'2016-03-14T14:11:25.000-04:00',
'2016-03-14T14:12:03.000-04:00',
'boston',
6
]
]
}),
('houshold_member_repeat',
{
'fields': [
'household_member_name',
'_parent_table_name',
'_parent_index'
],
'data': [
[
'peter',
'Grouped Repeatable Alias',
1
],
[
'kyle',
'Grouped Repeatable Alias',
2
],
[
'linda',
'Grouped Repeatable Alias',
2
],
[
'morty',
'Grouped Repeatable Alias',
3
],
[
'tony',
'Grouped Repeatable Alias',
4
],
[
'mary',
'Grouped Repeatable Alias',
4
],
[
'emma',
'Grouped Repeatable Alias',
5
],
[
'parker',
'Grouped Repeatable Alias',
5
],
[
'amadou',
'Grouped Repeatable Alias',
6
],
[
'esteban',
'Grouped Repeatable Alias',
6
],
[
'suzie',
'Grouped Repeatable Alias',
6
],
[
'fiona',
'Grouped Repeatable Alias',
6
],
[
'phillip',
'Grouped Repeatable Alias',
6
]
]
})
])
)
def test_export_with_choice_lists(self):
title, schemas, submissions = restaurant_profile
fp = FormPack(schemas, title)
self.assertEqual(len(fp[1].translations), 2)
# by default, exports use the question 'name' attribute
options = {'versions': 'rpV3'}
export = fp.export(**options).to_dict(submissions)['Restaurant profile']
self.assertEquals(export['fields'], ['restaurant_name',
'location',
'_location_latitude',
'_location_longitude',
'_location_altitude',
'_location_precision',
'eatery_type'])
self.assertEquals(export['data'], [['Taco Truck',
'13.42 -25.43',
'13.42',
'-25.43',
'',
'',
'takeaway'],
['Harvest',
'12.43 -24.53',
'12.43',
'-24.53',
'',
'',
'sit_down']])
# if a language is passed, fields with available translations
# are translated into that language
options['lang'] = fp[1].translations[0]
export = fp.export(**options).to_dict(submissions)['Restaurant profile']
self.assertEquals(export['data'], [['Taco Truck',
'13.42 -25.43',
'13.42',
'-25.43',
'',
'',
'take-away'],
['Harvest',
'12.43 -24.53',
'12.43',
'-24.53',
'',
'',
'sit down']])
options['lang'] = fp[1].translations[1]
export = fp.export(**options).to_dict(submissions)['Restaurant profile']
self.assertEquals(export['data'], [['Taco Truck',
'13.42 -25.43',
'13.42',
'-25.43',
'',
'',
'avec vente à emporter'],
['Harvest',
'12.43 -24.53',
'12.43',
'-24.53',
'',
'',
'traditionnel']])
def test_export_with_split_fields(self):
title, schemas, submissions = restaurant_profile
fp = FormPack(schemas, title)
options = {'versions': 'rpV4'}
export = fp.export(**options).to_dict(submissions)['Restaurant profile']
expected = {
'fields': [
'restaurant_name',
'location',
'_location_latitude',
'_location_longitude',
'_location_altitude',
'_location_precision',
'eatery_type',
'eatery_type/sit_down',
'eatery_type/takeaway',
],
'data': [
[
'Taco Truck',
'13.42 -25.43',
'13.42',
'-25.43',
'',
'',
'takeaway sit_down',
'1',
'1'
],
[
'Harvest',
'12.43 -24.53',
'12.43',
'-24.53',
'',
'',
'sit_down',
'1',
'0'
],
[
'Wololo',
'12.43 -24.54 1 0',
'12.43',
'-24.54',
'1',
'0',
'',
'0',
'0'
],
[
'Los pollos hermanos',
'12.43 -24.54 1',
'12.43',
'-24.54',
'1',
'',
'',
'',
''
]
]
}
self.assertEqual(export, expected)
options = {'versions': 'rpV4', "group_sep": "::",
'hierarchy_in_labels': True,
"lang": fp[-1].translations[1]}
export = fp.export(**options).to_dict(submissions)['Restaurant profile']
expected = {
'fields': [
'nom du restaurant',
'lieu',
'_lieu_latitude',
'_lieu_longitude',
'_lieu_altitude',
'_lieu_precision',
'type de restaurant',
'type de restaurant::traditionnel',
'type de restaurant::avec vente à emporter',
],
'data': [
[
'Taco Truck',
'13.42 -25.43',
'13.42',
'-25.43',
'',
'',
'avec vente à emporter traditionnel',
'1',
'1'
],
[
'Harvest',
'12.43 -24.53',
'12.43',
'-24.53',
'',
'',
'traditionnel',
'1',
'0'
],
[
'Wololo',
'12.43 -24.54 1 0',
'12.43',
'-24.54',
'1',
'0',
'',
'0',
'0'
],
[
'Los pollos hermanos',
'12.43 -24.54 1',
'12.43',
'-24.54',
'1',
'',
'',
'',
''
]
]
}
self.assertEqual(export, expected)
def data_by_identifiers(asset, field_names=None, submission_stream=None,
report_styles=None, lang=None, fields=None,
split_by=None):
if submission_stream is None:
_userform_id = asset.deployment.mongo_userform_id
submission_stream = get_instances_for_userform_id(_userform_id)
_versions = asset.deployed_versions
# need ability to look up deprecated IDs
_reversion_ids = dict([
(str(v._reversion_version_id), v.uid)
for v in _versions if v._reversion_version_id
])
schemas = [v.to_formpack_schema() for v in _versions]
_version_id = schemas[0]['version']
_version_id_key = schemas[0].get('version_id_key', '__version__')
def _inject_version_id(result):
if _version_id_key not in result:
result[_version_id_key] = _version_id
elif result[_version_id_key] in _reversion_ids:
result[_version_id_key] = _reversion_ids[result[_version_id_key]]
return result
submission_stream = (_inject_version_id(result)
for result in submission_stream)
pack = FormPack(versions=schemas, id_string=asset.uid)
_all_versions = pack.versions.keys()
report = pack.autoreport(versions=_all_versions)
fields_by_name = OrderedDict([
(field.name, field) for field in
pack.get_fields_for_versions(versions=_all_versions)
])
if field_names is None:
field_names = fields_by_name.keys()
if split_by and (split_by not in fields_by_name):
raise serializers.ValidationError(_("`split_by` field '{}' not found.").format(split_by))
if split_by and (fields_by_name[split_by].data_type != 'select_one'):
raise serializers.ValidationError(_("`split_by` field '{}' is not a select one question.").
format(split_by))
if report_styles is None:
report_styles = asset.report_styles
specified_styles = report_styles.get('specified', {})
kuids = report_styles.get('kuid_names', {})
def _stat_dict_to_array(stat, field_name):
freq = stat.pop('frequency', [])
if len(freq) > 0:
prcntg = stat.pop('percentage')
responses, frequencies = zip(*freq)
responses_percentage, percentages = zip(*prcntg)
if responses != responses_percentage:
raise ValueError("Frequency and percentage response lists for field '{}' mismatch."
.format(field_name))
stat.update({'responses': responses,
'frequencies': frequencies,
'percentages': percentages})
def _package_stat(field, _, stat, split_by):
identifier = kuids.get(field.name)
if not split_by:
_stat_dict_to_array(stat, field.name)
elif 'values' in stat:
for _, sub_stat in stat['values']:
_stat_dict_to_array(sub_stat, field.name)
return {
'name': field.name,
'row': {'type': fields_by_name.get(field.name).data_type},
'data': stat,
'kuid': identifier,
'style': specified_styles.get(identifier, {}),
}
return [_package_stat(*stat_tup, split_by=split_by) for
stat_tup in report.get_stats(submission_stream,
fields=field_names,
lang=lang,
split_by=split_by)
]
from formpack import FormPack
from formpack.fixtures import build_fixture
import time
a = time.time()
print('Loading fixtures')
title, schemas, submissions = build_fixture('grouped_repeatable')
b = time.time()
print(b - a, 's')
print('Loading schema')
fp = FormPack(schemas, title)
a = time.time()
print(a - b, 's')
options = { }
print('Python export')
export = fp.export(**options)
b = time.time()
print(b - a, 's')
print('xls export')
#export.to_xlsx('/tmp/foo.xlsx', submissions)
def test_disaggregate(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, split_by="when")
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6,
'values': [('Felipes',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 0),
('2003-01-01', 0)],
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 0.00),
('2003-01-01', 0.00)]}),
('The other one',
{'frequency': [('2001-01-01', 0),
('2002-01-01', 2),
('2003-01-01', 0)],
'percentage': [('2001-01-01', 0.00),
('2002-01-01', 33.33),
('2003-01-01', 0.00)]}),
('That one',
{'frequency': [('2001-01-01', 0),
('2002-01-01', 0),
('2003-01-01', 1)],
'percentage': [('2001-01-01', 0.00),
('2002-01-01', 0.00),
('2003-01-01', 16.67)]})]}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6,
'values': (('2001-01-01',
{'mean': 1.5,
'median': 1.5,
'mode': '*',
'stdev': 0.7071067811865476}),
('2002-01-01',
{'mean': 2.0, 'median': 2.0, 'mode': 2, 'stdev': 0.0}),
('2003-01-01',
{'mean': 1.0,
'median': 1,
'mode': '*',
'stdev': u'*'}))})]
for (i, stat) in enumerate(stats):
assert stat == expected[i]