def test_disaggregate_extended_fields(self): title, schemas, submissions = build_fixture( 'auto_report_extended_fields') fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions, split_by="when") assert stats.submissions_count == 22 stats = [(str(repr(field)), field_name, stats_dict) for field, field_name, stats_dict in stats] for stat in stats: stats_dict = dict(stat[2]) for value in stats_dict.get("values"): value_list = value[1] percentage_responses = [ x[0] for x in value_list.get("percentage") ] frequency_responses = [ x[0] for x in value_list.get("frequency") ] assert percentage_responses == frequency_responses assert percentage_responses[-1] == "..."
def test_rich_report(self): title, schemas, submissions = build_fixture('auto_report') fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions) assert stats.submissions_count == 6 stats = [(unicode(repr(f)), n, d) for f, n, d in stats] expected = [ ("<TextField name='restaurant_name' type='text'>", 'restaurant_name', {'frequency': [('Felipes', 2), ('The other one', 2), ('That one', 1)], 'not_provided': 1, 'percentage': [('Felipes', 33.33), ('The other one', 33.33), ('That one', 16.67)], 'provided': 5, 'show_graph': False, 'total_count': 6}), ("<FormGPSField name='location' type='geopoint'>", 'location', {'not_provided': 1, 'provided': 5, 'show_graph': False, 'total_count': 6}), ("<DateField name='when' type='date'>", 'when', {'frequency': [('2001-01-01', 2), ('2002-01-01', 2), ('2003-01-01', 1)], 'not_provided': 1, 'percentage': [('2001-01-01', 33.33), ('2002-01-01', 33.33), ('2003-01-01', 16.67)], 'provided': 5, 'show_graph': True, 'total_count': 6}), ("<NumField name='howmany' type='integer'>", 'howmany', {'mean': 1.6, 'median': 2, 'mode': 2, 'not_provided': 1, 'provided': 5, 'show_graph': False, 'stdev': 0.5477225575051661, 'total_count': 6} ) ] for (i, stat) in enumerate(stats): assert stat == expected[i]
def test_simple_report(self): title, schemas, submissions = build_fixture('restaurant_profile') fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions, lang='french') assert stats.submissions_count == 4 stats = [(str(repr(f)), n, d) for f, n, d in stats] expected = [ ("<TextField name='restaurant_name' type='text'>", 'nom du restaurant', { 'frequency': [('Taco Truck', 1), ('Harvest', 1), ('Wololo', 1), ('Los pollos hermanos', 1)], 'not_provided': 0, 'percentage': [('Taco Truck', 25.00), ('Harvest', 25.00), ('Wololo', 25.00), ('Los pollos hermanos', 25.00)], 'provided': 4, 'show_graph': False, 'total_count': 4 }), ("<FormGPSField name='location' type='geopoint'>", 'lieu', { 'not_provided': 0, 'provided': 4, 'show_graph': False, 'total_count': 4 }), ("<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>", 'type de restaurant', { 'frequency': [('traditionnel', 2), ('avec vente \xe0 emporter', 1)], 'not_provided': 1, 'percentage': [('traditionnel', 50.00), ('avec vente \xe0 emporter', 25.00)], 'provided': 3, 'show_graph': True, 'total_count': 4 }) ] for i, stat in enumerate(stats): assert stat == expected[i]
def test_stats_with_non_numeric_value_for_numeric_field(self): """ A string response to an integer question, for example, should not cause a crash; it should be treated as if no response was provided """ title = 'Just one number' schemas = [{ 'content': { 'survey': [{ 'type': 'integer', 'name': 'the_number', 'label': 'Enter the number!', }] } }] submissions = [ { 'the_number': 10 }, { 'the_number': 20 }, { 'the_number': 30 }, { 'the_number': 'oops!' }, ] fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions) assert stats.submissions_count == len(submissions) stats = [(str(repr(f)), n, d) for f, n, d in stats] expected = [( "<NumField name='the_number' type='integer'>", 'the_number', { 'mean': 20.0, 'median': 20, 'mode': '*', 'not_provided': 1, 'provided': 3, 'show_graph': False, 'stdev': 10.0, 'total_count': 4, }, )] for i, stat in enumerate(stats): assert stat == expected[i]
def test_submission_counts_match(): title, schemas, submissions = build_fixture('restaurant_profile') fp = FormPack(schemas, title) report = fp.autoreport(versions=fp.versions.keys()) stats = report.get_stats(submissions) assert stats.submissions_count == len(submissions) assert stats.submission_counts_by_version == { u'rpv1': 1, u'rpV2': 1, u'rpV3': 2, u'rpV4': 4, }
def test_simple_report(self): title, schemas, submissions = build_fixture('restaurant_profile') fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions, lang='french') assert stats.submissions_count == 4 stats = [(unicode(repr(f)), n, d) for f, n, d in stats] expected = [ ( "<TextField name='restaurant_name' type='text'>", 'nom du restaurant', {'frequency': [('Taco Truck', 1), ('Harvest', 1), ('Los pollos hermanos', 1), ('Wololo', 1)], 'not_provided': 0, 'percentage': [('Taco Truck', 25.00), ('Harvest', 25.00), ('Los pollos hermanos', 25.00), ('Wololo', 25.00)], 'provided': 4, 'show_graph': False, 'total_count': 4} ), ( "<FormGPSField name='location' type='geopoint'>", 'lieu', {'not_provided': 0, 'provided': 4, 'show_graph': False, 'total_count': 4} ), ( "<FormChoiceFieldWithMultipleSelect name='eatery_type' type='select_multiple'>", 'type de restaurant', {'frequency': [('traditionnel', 2), ('avec vente \xe0 emporter', 1)], 'not_provided': 1, 'percentage': [('traditionnel', 50.00), ('avec vente \xe0 emporter', 25.00)], 'provided': 3, 'show_graph': True, 'total_count': 4} ) ] for (i, stat) in enumerate(stats): assert stat == expected[i]
def test_disaggregate(self): title, schemas, submissions = build_fixture('auto_report') fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions, split_by="when") assert stats.submissions_count == 6 stats = [(unicode(repr(f)), n, d) for f, n, d in stats] expected = [ ("<TextField name='restaurant_name' type='text'>", 'restaurant_name', {'not_provided': 1, 'provided': 5, 'show_graph': False, 'total_count': 6, 'values': [('Felipes', {'frequency': [('2001-01-01', 2), ('2002-01-01', 0), ('2003-01-01', 0)], 'percentage': [('2001-01-01', 33.33), ('2002-01-01', 0.00), ('2003-01-01', 0.00)]}), ('The other one', {'frequency': [('2001-01-01', 0), ('2002-01-01', 2), ('2003-01-01', 0)], 'percentage': [('2001-01-01', 0.00), ('2002-01-01', 33.33), ('2003-01-01', 0.00)]}), ('That one', {'frequency': [('2001-01-01', 0), ('2002-01-01', 0), ('2003-01-01', 1)], 'percentage': [('2001-01-01', 0.00), ('2002-01-01', 0.00), ('2003-01-01', 16.67)]})]}), ("<FormGPSField name='location' type='geopoint'>", 'location', {'not_provided': 1, 'provided': 5, 'show_graph': False, 'total_count': 6}), ("<NumField name='howmany' type='integer'>", 'howmany', {'not_provided': 1, 'provided': 5, 'show_graph': False, 'total_count': 6, 'values': (('2001-01-01', {'mean': 1.5, 'median': 1.5, 'mode': '*', 'stdev': 0.7071067811865476}), ('2002-01-01', {'mean': 2.0, 'median': 2.0, 'mode': 2, 'stdev': 0.0}), ('2003-01-01', {'mean': 1.0, 'median': 1, 'mode': '*', 'stdev': u'*'}))})] for (i, stat) in enumerate(stats): assert stat == expected[i]
def test_simple_multi_version_report(self): title, schemas, submissions = build_fixture('site_inspection') fp = FormPack(schemas, title) report = fp.autoreport(versions=fp.versions.keys()) stats = report.get_stats(submissions) self.assertEqual(stats.submissions_count, 10) stats = [(repr(f), n, d) for f, n, d in stats] self.assertListEqual(stats, [ ( "<TextField name='inspector' type='text'>", u'inspector', {u'frequency': [(u'burger', 5), (u'clouseau', 5)], u'not_provided': 0, u'percentage': [(u'burger', 50.0), (u'clouseau', 50.0)], u'provided': 10, u'show_graph': False, u'total_count': 10} ), ( "<FormChoiceField name='did_you_find_the_site' type='select_one'>", u'did_you_find_the_site', {u'frequency': [(0, 4), (1, 4), (u'yes', 1), (u'no', 1)], u'not_provided': 0, u'percentage': [(0, 40.0), (1, 40.0), (u'yes', 10.0), (u'no', 10.0)], u'provided': 10, u'show_graph': True, u'total_count': 10} ), ( "<FormChoiceField name='was_there_damage_to_the_site' type='select_one'>", u'was_there_damage_to_the_site', {u'frequency': [(0, 2), (1, 2), (u'yes', 1)], u'not_provided': 5, u'percentage': [(0, 40.0), (1, 40.0), (u'yes', 20.0)], u'provided': 5, u'show_graph': True, u'total_count': 10} ), ( "<FormChoiceField name='was_there_damage_to_the_site_dupe' type='select_one'>", u'was_there_damage_to_the_site_dupe', {u'frequency': [(1, 1), (u'yes', 1)], u'not_provided': 8, u'percentage': [(1, 50.0), (u'yes', 50.0)], u'provided': 2, u'show_graph': True, u'total_count': 10} ), ( "<NumField name='ping' type='integer'>", u'ping', {u'mean': 238.4, u'median': 123, u'mode': u'*', u'not_provided': 5, u'provided': 5, u'show_graph': False, u'stdev': 255.77392361224003, u'total_count': 10} ), ( "<NumField name='rssi' type='integer'>", u'rssi', {u'mean': 63.8, u'median': u'65', u'mode': u'*', u'not_provided': 5, u'provided': 5, u'show_graph': False, u'stdev': 35.22357165308481, u'total_count': 10} ), ( "<FormChoiceField name='is_the_gate_secure' type='select_one'>", u'is_the_gate_secure', {u'frequency': [(0, 2), (1, 2), (u'no', 1)], u'not_provided': 5, u'percentage': [(0, 40.0), (1, 40.0), (u'no', 20.0)], u'provided': 5, u'show_graph': True, u'total_count': 10} ), ( "<FormChoiceField name='is_plant_life_encroaching' type='select_one'>", u'is_plant_life_encroaching', {u'frequency': [(0, 1), (1, 3), (u'yes', 1)], u'not_provided': 5, u'percentage': [(0, 20.0), (1, 60.0), (u'yes', 20.0)], u'provided': 5, u'show_graph': True, u'total_count': 10} ), ( "<FormChoiceField name='please_rate_the_impact_of_any_defects_observed' type='select_one'>", u'please_rate_the_impact_of_any_defects_observed', {u'frequency': [(u'moderate', 4), (u'severe', 3), (u'low', 3)], u'not_provided': 0, u'percentage': [(u'moderate', 40.0), (u'severe', 30.0), (u'low', 30.0)], u'provided': 10, u'show_graph': True, u'total_count': 10} ) ])
def data_by_identifiers(asset, field_names=None, submission_stream=None, report_styles=None, lang=None, fields=None, split_by=None): if submission_stream is None: _userform_id = asset.deployment.mongo_userform_id submission_stream = get_instances_for_userform_id(_userform_id) _versions = asset.deployed_versions # need ability to look up deprecated IDs _reversion_ids = dict([(str(v._reversion_version_id), v.uid) for v in _versions if v._reversion_version_id]) schemas = [v.to_formpack_schema() for v in _versions] _version_id = schemas[0]['version'] _version_id_key = schemas[0].get('version_id_key', '__version__') def _inject_version_id(result): if _version_id_key not in result: result[_version_id_key] = _version_id elif result[_version_id_key] in _reversion_ids: result[_version_id_key] = _reversion_ids[result[_version_id_key]] return result submission_stream = (_inject_version_id(result) for result in submission_stream) pack = FormPack(versions=schemas, id_string=asset.uid) _all_versions = pack.versions.keys() report = pack.autoreport(versions=_all_versions) fields_by_name = OrderedDict([ (field.name, field) for field in pack.get_fields_for_versions(versions=_all_versions) ]) if field_names is None: field_names = fields_by_name.keys() if split_by and (split_by not in fields_by_name): raise serializers.ValidationError( _("`split_by` field '{}' not found.").format(split_by)) if split_by and (fields_by_name[split_by].data_type != 'select_one'): raise serializers.ValidationError( _("`split_by` field '{}' is not a select one question.").format( split_by)) if report_styles is None: report_styles = asset.report_styles specified_styles = report_styles.get('specified', {}) kuids = report_styles.get('kuid_names', {}) def _stat_dict_to_array(stat, field_name): freq = stat.pop('frequency', []) if len(freq) > 0: prcntg = stat.pop('percentage') responses, frequencies = zip(*freq) responses_percentage, percentages = zip(*prcntg) if responses != responses_percentage: raise ValueError( "Frequency and percentage response lists for field '{}' mismatch." .format(field_name)) stat.update({ 'responses': responses, 'frequencies': frequencies, 'percentages': percentages }) def _package_stat(field, _, stat, split_by): identifier = kuids.get(field.name) if not split_by: _stat_dict_to_array(stat, field.name) elif 'values' in stat: for _, sub_stat in stat['values']: _stat_dict_to_array(sub_stat, field.name) return { 'name': field.name, 'row': { 'type': fields_by_name.get(field.name).data_type }, 'data': stat, 'kuid': identifier, 'style': specified_styles.get(identifier, {}), } return [ _package_stat(*stat_tup, split_by=split_by) for stat_tup in report.get_stats(submission_stream, fields=field_names, lang=lang, split_by=split_by) ]
def test_rich_report_or_other(self): title, schemas, submissions = build_fixture('or_other') fp = FormPack(schemas, title) report = fp.autoreport() stats = report.get_stats(submissions) assert stats.submissions_count == 3 stats = [(str(repr(f)), n, d) for f, n, d in stats] expected = [ ( "<FormChoiceField name='fav_emperor' type='select_one'>", 'fav_emperor', { 'total_count': 3, 'not_provided': 0, 'provided': 3, 'show_graph': True, 'frequency': [('other', 2), ('augustus', 1)], 'percentage': [('other', 66.67), ('augustus', 33.33)], }, ), ( "<TextField name='fav_emperor_other' type='text'>", 'fav_emperor_other', { 'total_count': 3, 'not_provided': 1, 'provided': 2, 'show_graph': False, 'frequency': [('Nero', 1), ('Marcus Aurelius', 1)], 'percentage': [('Nero', 33.33), ('Marcus Aurelius', 33.33)], }, ), ( "<FormChoiceFieldWithMultipleSelect name='fav_emperors' type='select_multiple'>", 'fav_emperors', { 'total_count': 3, 'not_provided': 0, 'provided': 3, 'show_graph': True, 'frequency': [ ('julius', 2), ('caligula', 1), ('other', 1), ('tiberius', 1), ('augustus', 1), ], 'percentage': [ ('julius', 66.67), ('caligula', 33.33), ('other', 33.33), ('tiberius', 33.33), ('augustus', 33.33), ], }, ), ( "<TextField name='fav_emperors_other' type='text'>", 'fav_emperors_other', { 'total_count': 3, 'not_provided': 2, 'provided': 1, 'show_graph': False, 'frequency': [('Commodus', 1)], 'percentage': [('Commodus', 33.33)], }, ), ] for i, stat in enumerate(stats): assert stat == expected[i]
def data_by_identifiers(asset, field_names=None, submission_stream=None, report_styles=None, lang=None, fields=None, split_by=None): if submission_stream is None: _userform_id = asset.deployment.mongo_userform_id submission_stream = get_instances_for_userform_id(_userform_id) _versions = asset.deployed_versions # need ability to look up deprecated IDs _reversion_ids = dict([ (str(v._reversion_version_id), v.uid) for v in _versions if v._reversion_version_id ]) schemas = [v.to_formpack_schema() for v in _versions] _version_id = schemas[0]['version'] _version_id_key = schemas[0].get('version_id_key', '__version__') def _inject_version_id(result): if _version_id_key not in result: result[_version_id_key] = _version_id elif result[_version_id_key] in _reversion_ids: result[_version_id_key] = _reversion_ids[result[_version_id_key]] return result submission_stream = (_inject_version_id(result) for result in submission_stream) pack = FormPack(versions=schemas, id_string=asset.uid) _all_versions = pack.versions.keys() report = pack.autoreport(versions=_all_versions) fields_by_name = OrderedDict([ (field.name, field) for field in pack.get_fields_for_versions(versions=_all_versions) ]) if field_names is None: field_names = fields_by_name.keys() if split_by and (split_by not in fields_by_name): raise serializers.ValidationError(_("`split_by` field '{}' not found.").format(split_by)) if split_by and (fields_by_name[split_by].data_type != 'select_one'): raise serializers.ValidationError(_("`split_by` field '{}' is not a select one question."). format(split_by)) if report_styles is None: report_styles = asset.report_styles specified_styles = report_styles.get('specified', {}) kuids = report_styles.get('kuid_names', {}) def _stat_dict_to_array(stat, field_name): freq = stat.pop('frequency', []) if len(freq) > 0: prcntg = stat.pop('percentage') responses, frequencies = zip(*freq) responses_percentage, percentages = zip(*prcntg) if responses != responses_percentage: raise ValueError("Frequency and percentage response lists for field '{}' mismatch." .format(field_name)) stat.update({'responses': responses, 'frequencies': frequencies, 'percentages': percentages}) def _package_stat(field, _, stat, split_by): identifier = kuids.get(field.name) if not split_by: _stat_dict_to_array(stat, field.name) elif 'values' in stat: for _, sub_stat in stat['values']: _stat_dict_to_array(sub_stat, field.name) return { 'name': field.name, 'row': {'type': fields_by_name.get(field.name).data_type}, 'data': stat, 'kuid': identifier, 'style': specified_styles.get(identifier, {}), } return [_package_stat(*stat_tup, split_by=split_by) for stat_tup in report.get_stats(submission_stream, fields=field_names, lang=lang, split_by=split_by) ]