def test_disaggregate_extended_fields(self):
title, schemas, submissions = build_fixture(
'auto_report_extended_fields')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, split_by="when")
assert stats.submissions_count == 22
stats = [(str(repr(field)), field_name, stats_dict)
for field, field_name, stats_dict in stats]
for stat in stats:
stats_dict = dict(stat[2])
for value in stats_dict.get("values"):
value_list = value[1]
percentage_responses = [
x[0] for x in value_list.get("percentage")
]
frequency_responses = [
x[0] for x in value_list.get("frequency")
]
assert percentage_responses == frequency_responses
assert percentage_responses[-1] == "..."
def test_rich_report(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'frequency': [('Felipes', 2),
('The other one', 2),
('That one', 1)],
'not_provided': 1,
'percentage': [('Felipes', 33.33),
('The other one', 33.33),
('That one', 16.67)],
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<DateField name='when' type='date'>",
'when',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 2),
('2003-01-01', 1)],
'not_provided': 1,
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 33.33),
('2003-01-01', 16.67)],
'provided': 5,
'show_graph': True,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'mean': 1.6,
'median': 2,
'mode': 2,
'not_provided': 1,
'provided': 5,
'show_graph': False,
'stdev': 0.5477225575051661,
'total_count': 6}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_rich_report(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'frequency': [('Felipes', 2),
('The other one', 2),
('That one', 1)],
'not_provided': 1,
'percentage': [('Felipes', 33.33),
('The other one', 33.33),
('That one', 16.67)],
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<DateField name='when' type='date'>",
'when',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 2),
('2003-01-01', 1)],
'not_provided': 1,
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 33.33),
('2003-01-01', 16.67)],
'provided': 5,
'show_graph': True,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'mean': 1.6,
'median': 2,
'mode': 2,
'not_provided': 1,
'provided': 5,
'show_graph': False,
'stdev': 0.5477225575051661,
'total_count': 6}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_simple_report(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, lang='french')
assert stats.submissions_count == 4
stats = [(str(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'nom du restaurant', {
'frequency': [('Taco Truck', 1), ('Harvest', 1),
('Wololo', 1), ('Los pollos hermanos', 1)],
'not_provided':
0,
'percentage': [('Taco Truck', 25.00), ('Harvest', 25.00),
('Wololo', 25.00),
('Los pollos hermanos', 25.00)],
'provided':
4,
'show_graph':
False,
'total_count':
4
}),
("<FormGPSField name='location' type='geopoint'>", 'lieu', {
'not_provided': 0,
'provided': 4,
'show_graph': False,
'total_count': 4
}),
("<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>",
'type de restaurant', {
'frequency': [('traditionnel', 2),
('avec vente \xe0 emporter', 1)],
'not_provided':
1,
'percentage': [('traditionnel', 50.00),
('avec vente \xe0 emporter', 25.00)],
'provided':
3,
'show_graph':
True,
'total_count':
4
})
]
for i, stat in enumerate(stats):
assert stat == expected[i]
def test_stats_with_non_numeric_value_for_numeric_field(self):
"""
A string response to an integer question, for example, should not cause
a crash; it should be treated as if no response was provided
"""
title = 'Just one number'
schemas = [{
'content': {
'survey': [{
'type': 'integer',
'name': 'the_number',
'label': 'Enter the number!',
}]
}
}]
submissions = [
{
'the_number': 10
},
{
'the_number': 20
},
{
'the_number': 30
},
{
'the_number': 'oops!'
},
]
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == len(submissions)
stats = [(str(repr(f)), n, d) for f, n, d in stats]
expected = [(
"<NumField name='the_number' type='integer'>",
'the_number',
{
'mean': 20.0,
'median': 20,
'mode': '*',
'not_provided': 1,
'provided': 3,
'show_graph': False,
'stdev': 10.0,
'total_count': 4,
},
)]
for i, stat in enumerate(stats):
assert stat == expected[i]
def test_submission_counts_match():
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport(versions=fp.versions.keys())
stats = report.get_stats(submissions)
assert stats.submissions_count == len(submissions)
assert stats.submission_counts_by_version == {
u'rpv1': 1,
u'rpV2': 1,
u'rpV3': 2,
u'rpV4': 4,
}
def test_simple_report(self):
title, schemas, submissions = build_fixture('restaurant_profile')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, lang='french')
assert stats.submissions_count == 4
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
(
"<TextField name='restaurant_name' type='text'>",
'nom du restaurant',
{'frequency': [('Taco Truck', 1),
('Harvest', 1),
('Los pollos hermanos', 1),
('Wololo', 1)],
'not_provided': 0,
'percentage': [('Taco Truck', 25.00),
('Harvest', 25.00),
('Los pollos hermanos', 25.00),
('Wololo', 25.00)],
'provided': 4,
'show_graph': False,
'total_count': 4}
),
(
"<FormGPSField name='location' type='geopoint'>",
'lieu',
{'not_provided': 0,
'provided': 4,
'show_graph': False,
'total_count': 4}
),
(
"<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>",
'type de restaurant',
{'frequency': [('traditionnel', 2), ('avec vente \xe0 emporter', 1)],
'not_provided': 1,
'percentage': [('traditionnel', 50.00),
('avec vente \xe0 emporter', 25.00)],
'provided': 3,
'show_graph': True,
'total_count': 4}
)
]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_disaggregate(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, split_by="when")
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6,
'values': [('Felipes',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 0),
('2003-01-01', 0)],
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 0.00),
('2003-01-01', 0.00)]}),
('The other one',
{'frequency': [('2001-01-01', 0),
('2002-01-01', 2),
('2003-01-01', 0)],
'percentage': [('2001-01-01', 0.00),
('2002-01-01', 33.33),
('2003-01-01', 0.00)]}),
('That one',
{'frequency': [('2001-01-01', 0),
('2002-01-01', 0),
('2003-01-01', 1)],
'percentage': [('2001-01-01', 0.00),
('2002-01-01', 0.00),
('2003-01-01', 16.67)]})]}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6,
'values': (('2001-01-01',
{'mean': 1.5,
'median': 1.5,
'mode': '*',
'stdev': 0.7071067811865476}),
('2002-01-01',
{'mean': 2.0, 'median': 2.0, 'mode': 2, 'stdev': 0.0}),
('2003-01-01',
{'mean': 1.0,
'median': 1,
'mode': '*',
'stdev': u'*'}))})]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def test_simple_multi_version_report(self):
title, schemas, submissions = build_fixture('site_inspection')
fp = FormPack(schemas, title)
report = fp.autoreport(versions=fp.versions.keys())
stats = report.get_stats(submissions)
self.assertEqual(stats.submissions_count, 10)
stats = [(repr(f), n, d) for f, n, d in stats]
self.assertListEqual(stats, [
(
"<TextField name='inspector' type='text'>",
u'inspector',
{u'frequency': [(u'burger', 5), (u'clouseau', 5)],
u'not_provided': 0,
u'percentage': [(u'burger', 50.0), (u'clouseau', 50.0)],
u'provided': 10,
u'show_graph': False,
u'total_count': 10}
),
(
"<FormChoiceField name='did_you_find_the_site' type='select_one'>",
u'did_you_find_the_site',
{u'frequency': [(0, 4), (1, 4), (u'yes', 1), (u'no', 1)],
u'not_provided': 0,
u'percentage': [(0, 40.0),
(1, 40.0),
(u'yes', 10.0),
(u'no', 10.0)],
u'provided': 10,
u'show_graph': True,
u'total_count': 10}
),
(
"<FormChoiceField name='was_there_damage_to_the_site' type='select_one'>",
u'was_there_damage_to_the_site',
{u'frequency': [(0, 2), (1, 2), (u'yes', 1)],
u'not_provided': 5,
u'percentage': [(0, 40.0), (1, 40.0), (u'yes', 20.0)],
u'provided': 5,
u'show_graph': True,
u'total_count': 10}
),
(
"<FormChoiceField name='was_there_damage_to_the_site_dupe' type='select_one'>",
u'was_there_damage_to_the_site_dupe',
{u'frequency': [(1, 1), (u'yes', 1)],
u'not_provided': 8,
u'percentage': [(1, 50.0), (u'yes', 50.0)],
u'provided': 2,
u'show_graph': True,
u'total_count': 10}
),
(
"<NumField name='ping' type='integer'>",
u'ping',
{u'mean': 238.4,
u'median': 123,
u'mode': u'*',
u'not_provided': 5,
u'provided': 5,
u'show_graph': False,
u'stdev': 255.77392361224003,
u'total_count': 10}
),
(
"<NumField name='rssi' type='integer'>",
u'rssi',
{u'mean': 63.8,
u'median': u'65',
u'mode': u'*',
u'not_provided': 5,
u'provided': 5,
u'show_graph': False,
u'stdev': 35.22357165308481,
u'total_count': 10}
),
(
"<FormChoiceField name='is_the_gate_secure' type='select_one'>",
u'is_the_gate_secure',
{u'frequency': [(0, 2), (1, 2), (u'no', 1)],
u'not_provided': 5,
u'percentage': [(0, 40.0), (1, 40.0), (u'no', 20.0)],
u'provided': 5,
u'show_graph': True,
u'total_count': 10}
),
(
"<FormChoiceField name='is_plant_life_encroaching' type='select_one'>",
u'is_plant_life_encroaching',
{u'frequency': [(0, 1), (1, 3), (u'yes', 1)],
u'not_provided': 5,
u'percentage': [(0, 20.0), (1, 60.0), (u'yes', 20.0)],
u'provided': 5,
u'show_graph': True,
u'total_count': 10}
),
(
"<FormChoiceField name='please_rate_the_impact_of_any_defects_observed' type='select_one'>",
u'please_rate_the_impact_of_any_defects_observed',
{u'frequency': [(u'moderate', 4), (u'severe', 3), (u'low', 3)],
u'not_provided': 0,
u'percentage': [(u'moderate', 40.0),
(u'severe', 30.0),
(u'low', 30.0)],
u'provided': 10,
u'show_graph': True,
u'total_count': 10}
)
])
def data_by_identifiers(asset,
field_names=None,
submission_stream=None,
report_styles=None,
lang=None,
fields=None,
split_by=None):
if submission_stream is None:
_userform_id = asset.deployment.mongo_userform_id
submission_stream = get_instances_for_userform_id(_userform_id)
_versions = asset.deployed_versions
# need ability to look up deprecated IDs
_reversion_ids = dict([(str(v._reversion_version_id), v.uid)
for v in _versions if v._reversion_version_id])
schemas = [v.to_formpack_schema() for v in _versions]
_version_id = schemas[0]['version']
_version_id_key = schemas[0].get('version_id_key', '__version__')
def _inject_version_id(result):
if _version_id_key not in result:
result[_version_id_key] = _version_id
elif result[_version_id_key] in _reversion_ids:
result[_version_id_key] = _reversion_ids[result[_version_id_key]]
return result
submission_stream = (_inject_version_id(result)
for result in submission_stream)
pack = FormPack(versions=schemas, id_string=asset.uid)
_all_versions = pack.versions.keys()
report = pack.autoreport(versions=_all_versions)
fields_by_name = OrderedDict([
(field.name, field)
for field in pack.get_fields_for_versions(versions=_all_versions)
])
if field_names is None:
field_names = fields_by_name.keys()
if split_by and (split_by not in fields_by_name):
raise serializers.ValidationError(
_("`split_by` field '{}' not found.").format(split_by))
if split_by and (fields_by_name[split_by].data_type != 'select_one'):
raise serializers.ValidationError(
_("`split_by` field '{}' is not a select one question.").format(
split_by))
if report_styles is None:
report_styles = asset.report_styles
specified_styles = report_styles.get('specified', {})
kuids = report_styles.get('kuid_names', {})
def _stat_dict_to_array(stat, field_name):
freq = stat.pop('frequency', [])
if len(freq) > 0:
prcntg = stat.pop('percentage')
responses, frequencies = zip(*freq)
responses_percentage, percentages = zip(*prcntg)
if responses != responses_percentage:
raise ValueError(
"Frequency and percentage response lists for field '{}' mismatch."
.format(field_name))
stat.update({
'responses': responses,
'frequencies': frequencies,
'percentages': percentages
})
def _package_stat(field, _, stat, split_by):
identifier = kuids.get(field.name)
if not split_by:
_stat_dict_to_array(stat, field.name)
elif 'values' in stat:
for _, sub_stat in stat['values']:
_stat_dict_to_array(sub_stat, field.name)
return {
'name': field.name,
'row': {
'type': fields_by_name.get(field.name).data_type
},
'data': stat,
'kuid': identifier,
'style': specified_styles.get(identifier, {}),
}
return [
_package_stat(*stat_tup, split_by=split_by)
for stat_tup in report.get_stats(submission_stream,
fields=field_names,
lang=lang,
split_by=split_by)
]
def test_rich_report_or_other(self):
title, schemas, submissions = build_fixture('or_other')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions)
assert stats.submissions_count == 3
stats = [(str(repr(f)), n, d) for f, n, d in stats]
expected = [
(
"<FormChoiceField name='fav_emperor' type='select_one'>",
'fav_emperor',
{
'total_count': 3,
'not_provided': 0,
'provided': 3,
'show_graph': True,
'frequency': [('other', 2), ('augustus', 1)],
'percentage': [('other', 66.67), ('augustus', 33.33)],
},
),
(
"<TextField name='fav_emperor_other' type='text'>",
'fav_emperor_other',
{
'total_count': 3,
'not_provided': 1,
'provided': 2,
'show_graph': False,
'frequency': [('Nero', 1), ('Marcus Aurelius', 1)],
'percentage': [('Nero', 33.33),
('Marcus Aurelius', 33.33)],
},
),
(
"<FormChoiceFieldWithMultipleSelect name='fav_emperors' type='select_multiple'>",
'fav_emperors',
{
'total_count':
3,
'not_provided':
0,
'provided':
3,
'show_graph':
True,
'frequency': [
('julius', 2),
('caligula', 1),
('other', 1),
('tiberius', 1),
('augustus', 1),
],
'percentage': [
('julius', 66.67),
('caligula', 33.33),
('other', 33.33),
('tiberius', 33.33),
('augustus', 33.33),
],
},
),
(
"<TextField name='fav_emperors_other' type='text'>",
'fav_emperors_other',
{
'total_count': 3,
'not_provided': 2,
'provided': 1,
'show_graph': False,
'frequency': [('Commodus', 1)],
'percentage': [('Commodus', 33.33)],
},
),
]
for i, stat in enumerate(stats):
assert stat == expected[i]
def test_disaggregate(self):
title, schemas, submissions = build_fixture('auto_report')
fp = FormPack(schemas, title)
report = fp.autoreport()
stats = report.get_stats(submissions, split_by="when")
assert stats.submissions_count == 6
stats = [(unicode(repr(f)), n, d) for f, n, d in stats]
expected = [
("<TextField name='restaurant_name' type='text'>",
'restaurant_name',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6,
'values': [('Felipes',
{'frequency': [('2001-01-01', 2),
('2002-01-01', 0),
('2003-01-01', 0)],
'percentage': [('2001-01-01', 33.33),
('2002-01-01', 0.00),
('2003-01-01', 0.00)]}),
('The other one',
{'frequency': [('2001-01-01', 0),
('2002-01-01', 2),
('2003-01-01', 0)],
'percentage': [('2001-01-01', 0.00),
('2002-01-01', 33.33),
('2003-01-01', 0.00)]}),
('That one',
{'frequency': [('2001-01-01', 0),
('2002-01-01', 0),
('2003-01-01', 1)],
'percentage': [('2001-01-01', 0.00),
('2002-01-01', 0.00),
('2003-01-01', 16.67)]})]}),
("<FormGPSField name='location' type='geopoint'>",
'location',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6}),
("<NumField name='howmany' type='integer'>",
'howmany',
{'not_provided': 1,
'provided': 5,
'show_graph': False,
'total_count': 6,
'values': (('2001-01-01',
{'mean': 1.5,
'median': 1.5,
'mode': '*',
'stdev': 0.7071067811865476}),
('2002-01-01',
{'mean': 2.0, 'median': 2.0, 'mode': 2, 'stdev': 0.0}),
('2003-01-01',
{'mean': 1.0,
'median': 1,
'mode': '*',
'stdev': u'*'}))})]
for (i, stat) in enumerate(stats):
assert stat == expected[i]
def data_by_identifiers(asset, field_names=None, submission_stream=None,
report_styles=None, lang=None, fields=None,
split_by=None):
if submission_stream is None:
_userform_id = asset.deployment.mongo_userform_id
submission_stream = get_instances_for_userform_id(_userform_id)
_versions = asset.deployed_versions
# need ability to look up deprecated IDs
_reversion_ids = dict([
(str(v._reversion_version_id), v.uid)
for v in _versions if v._reversion_version_id
])
schemas = [v.to_formpack_schema() for v in _versions]
_version_id = schemas[0]['version']
_version_id_key = schemas[0].get('version_id_key', '__version__')
def _inject_version_id(result):
if _version_id_key not in result:
result[_version_id_key] = _version_id
elif result[_version_id_key] in _reversion_ids:
result[_version_id_key] = _reversion_ids[result[_version_id_key]]
return result
submission_stream = (_inject_version_id(result)
for result in submission_stream)
pack = FormPack(versions=schemas, id_string=asset.uid)
_all_versions = pack.versions.keys()
report = pack.autoreport(versions=_all_versions)
fields_by_name = OrderedDict([
(field.name, field) for field in
pack.get_fields_for_versions(versions=_all_versions)
])
if field_names is None:
field_names = fields_by_name.keys()
if split_by and (split_by not in fields_by_name):
raise serializers.ValidationError(_("`split_by` field '{}' not found.").format(split_by))
if split_by and (fields_by_name[split_by].data_type != 'select_one'):
raise serializers.ValidationError(_("`split_by` field '{}' is not a select one question.").
format(split_by))
if report_styles is None:
report_styles = asset.report_styles
specified_styles = report_styles.get('specified', {})
kuids = report_styles.get('kuid_names', {})
def _stat_dict_to_array(stat, field_name):
freq = stat.pop('frequency', [])
if len(freq) > 0:
prcntg = stat.pop('percentage')
responses, frequencies = zip(*freq)
responses_percentage, percentages = zip(*prcntg)
if responses != responses_percentage:
raise ValueError("Frequency and percentage response lists for field '{}' mismatch."
.format(field_name))
stat.update({'responses': responses,
'frequencies': frequencies,
'percentages': percentages})
def _package_stat(field, _, stat, split_by):
identifier = kuids.get(field.name)
if not split_by:
_stat_dict_to_array(stat, field.name)
elif 'values' in stat:
for _, sub_stat in stat['values']:
_stat_dict_to_array(sub_stat, field.name)
return {
'name': field.name,
'row': {'type': fields_by_name.get(field.name).data_type},
'data': stat,
'kuid': identifier,
'style': specified_styles.get(identifier, {}),
}
return [_package_stat(*stat_tup, split_by=split_by) for
stat_tup in report.get_stats(submission_stream,
fields=field_names,
lang=lang,
split_by=split_by)
]
