def test_csv_export(self):
"""
Test CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
cursor = csv_df_builder._query_data()
return [d for d in csv_df_builder._format_for_dataframe(cursor)]
def to_flat_csv_export(self, path, data, username, id_string, filter_query,
**kwargs):
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
start = kwargs.get('start')
end = kwargs.get('end')
dataview = kwargs.get('dataview')
xform = kwargs.get('xform')
options = kwargs.get('options')
total_records = kwargs.get('total_records')
win_excel_utf8 = options.get('win_excel_utf8') if options else False
csv_builder = CSVDataFrameBuilder(username,
id_string,
filter_query,
self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES,
self.BINARY_SELECT_MULTIPLES,
start,
end,
self.TRUNCATE_GROUP_TITLE,
xform,
self.INCLUDE_LABELS,
self.INCLUDE_LABELS_ONLY,
self.INCLUDE_IMAGES,
self.INCLUDE_HXL,
win_excel_utf8=win_excel_utf8,
total_records=total_records)
csv_builder.export_to(path, dataview=dataview)
def test_csv_export(self):
self._publish_single_level_repeat_form()
# submit 7 instances
for i in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for i in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(
len(header), 17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_split_select_multiples_values(self):
"""
Test select multiples choices are split and their values as the data.
"""
self._publish_nested_repeats_form()
self._submit_fixture_instance("nested_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
# pylint: disable=protected-access
cursor = [k for k in csv_df_builder._query_data()]
record = cursor[0]
select_multiples = \
CSVDataFrameBuilder._collect_select_multiples(self.xform)
result = CSVDataFrameBuilder._split_select_multiples(
record, select_multiples, value_select_multiples=True)
expected_result = {
u'web_browsers/ie': u'ie',
u'web_browsers/safari': u'safari',
u'web_browsers/firefox': None,
u'web_browsers/chrome': None
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in list(result)
if key in list(expected_result)])
self.assertEqual(expected_result, result)
def test_csv_export(self):
self._publish_single_level_repeat_form()
# submit 7 instances
for i in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for i in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
def to_flat_csv_export(self, path, data, username, id_string,
filter_query, **kwargs):
"""
Generates a flattened CSV file for submitted data.
"""
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
start = kwargs.get('start')
end = kwargs.get('end')
dataview = kwargs.get('dataview')
xform = kwargs.get('xform')
options = kwargs.get('options')
total_records = kwargs.get('total_records')
win_excel_utf8 = options.get('win_excel_utf8') if options else False
index_tags = options.get(REPEAT_INDEX_TAGS, self.REPEAT_INDEX_TAGS)
show_choice_labels = options.get('show_choice_labels', False)
language = options.get('language')
csv_builder = CSVDataFrameBuilder(
username, id_string, filter_query, self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES,
start, end, self.TRUNCATE_GROUP_TITLE, xform,
self.INCLUDE_LABELS, self.INCLUDE_LABELS_ONLY, self.INCLUDE_IMAGES,
self.INCLUDE_HXL, win_excel_utf8=win_excel_utf8,
total_records=total_records, index_tags=index_tags,
value_select_multiples=self.VALUE_SELECT_MULTIPLES,
show_choice_labels=show_choice_labels, language=language)
csv_builder.export_to(path, dataview=dataview)
def test_windows_excel_compatible_csv_export(self):
"""
Test window excel compatible CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False,
win_excel_utf8=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
self.assertEqual(b'\xef\xbb\xbfname', header[0].encode('utf-8'))
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_dataframe_export_to(self):
self._publish_nested_repeats_form()
self._submit_fixture_instance("nested_repeats",
"01",
submission_time=self._submission_time)
self._submit_fixture_instance("nested_repeats",
"02",
submission_time=self._submission_time)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_fixture_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "fixtures",
"nested_repeats", "nested_repeats.csv")
temp_file.close()
fixture, output = '', ''
with open(csv_fixture_path) as f:
fixture = f.read()
with open(temp_file.name) as f:
output = f.read()
os.unlink(temp_file.name)
self.assertEqual(fixture, output)
def test_csv_export_with_labels(self):
"""
Test CSV export with labels.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received'
]
self.assertEqual(expected_header, header)
labels = csv_reader.next()
self.assertEqual(len(labels),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_labels = [
'Name', 'age', 'Do you have kids?', 'Kids Name', 'Kids Age',
'Kids Name', 'Kids Age', '5. Record your GPS coordinates.',
'_gps_latitude', '_gps_longitude', '_gps_altitude',
'_gps_precision', 'web_browsers/Mozilla Firefox',
'web_browsers/Google Chrome', 'web_browsers/Internet Explorer',
'web_browsers/Safari', 'instanceID', '_id', '_uuid',
'_submission_time', '_tags', '_notes', '_version', '_duration',
'_submitted_by', '_total_media', '_media_count',
'_media_all_received'
]
self.assertEqual(expected_labels, labels)
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def to_flat_csv_export(
self, path, data, username, id_string, filter_query,
start=None, end=None):
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(
username, id_string, filter_query, self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES, self.BINARY_SELECT_MULTIPLES,
start, end, self.TRUNCATE_GROUP_TITLE
)
csv_builder.export_to(path)
def test_no_split_select_multiples(self, mock_query_data):
"""
Test selct multiples are not split within repeats.
"""
md_xform = """
| survey |
| | type | name | label |
| | text | name | Name |
| | integer | age | Age |
| | begin repeat | browser_use | Browser Use |
| | integer | year | Year |
| | select_multiple browsers | browsers | Browsers |
| | end repeat | | |
| choices |
| | list name | name | label |
| | browsers | firefox | Firefox |
| | browsers | chrome | Chrome |
| | browsers | ie | Internet Explorer |
| | browsers | safari | Safari |
"""
xform = self._publish_markdown(md_xform, self.user, id_string='b')
data = [{
'name': 'Tom',
'age': 23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers': 'firefox safari'
}, {
'browser_use/year': '2011',
'browser_use/browsers': 'firefox chrome'
}]
}] # yapf: disable
mock_query_data.return_value = data
csv_df_builder = CSVDataFrameBuilder(self.user.username,
xform.id_string,
split_select_multiples=False,
include_images=False)
# pylint: disable=protected-access
cursor = [row for row in csv_df_builder._query_data()]
result = [k for k in csv_df_builder._format_for_dataframe(cursor)]
expected_result = [{
'name': 'Tom',
'age': 23,
'browser_use[1]/year': '2010',
'browser_use[1]/browsers': 'firefox safari',
'browser_use[2]/year': '2011',
'browser_use[2]/browsers': 'firefox chrome'
}]
self.maxDiff = None
self.assertEqual(expected_result, result)
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {
'xform_id__in':
list(xform.mergedxform.xforms.values_list('pk', flat=True))
}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
csv_df_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string,
include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data)
return self._get_streaming_response(data)
return Response(data)
def data(self, request, **kwargs):
self.object = self.get_object()
# get greater than value and cast it to an int
gt = request.query_params.get('gt_id')
gt = gt and parse_int(gt)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
qs_kwargs = {'xform_id': xform.pk}
if gt:
qs_kwargs.update({'id__gt': gt})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
length = self.total_count = instances.count()
if count:
return Response({'count': self.total_count})
if should_paginate:
instances = self.paginate_queryset(instances)
length = 1 + self.paginator.page.end_index(
) - self.paginator.page.start_index()
csv_df_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string,
include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data,
key_replacement_obj={
'pattern': r"(/|-|\[|\])",
"replacer": r"_"
})
return self._get_streaming_response(data, length)
return Response(data)
def data(self, request, **kwargs):
"""
Streams submission data response matching uuid in the request.
"""
self.object = self.get_object()
# get greater than value and cast it to an int
gt_id = request.query_params.get('gt_id')
gt_id = gt_id and parse_int(gt_id)
count = request.query_params.get('count')
pagination_keys = [
self.paginator.page_query_param,
self.paginator.page_size_query_param
]
query_param_keys = request.query_params
should_paginate = any([k in query_param_keys for k in pagination_keys])
data = []
if isinstance(self.object.content_object, XForm):
if not self.object.active:
return Response(status=status.HTTP_404_NOT_FOUND)
xform = self.object.content_object
if xform.is_merged_dataset:
qs_kwargs = {'xform_id__in': list(
xform.mergedxform.xforms.values_list('pk', flat=True))}
else:
qs_kwargs = {'xform_id': xform.pk}
if gt_id:
qs_kwargs.update({'id__gt': gt_id})
instances = Instance.objects.filter(**qs_kwargs).order_by('pk')
if count:
return Response({'count': instances.count()})
if should_paginate:
instances = self.paginate_queryset(instances)
csv_df_builder = CSVDataFrameBuilder(
xform.user.username, xform.id_string, include_images=False)
data = csv_df_builder._format_for_dataframe(
DataInstanceSerializer(instances, many=True).data)
return self._get_streaming_response(data)
return Response(data)
def to_flat_csv_export(self,
path,
data,
username,
id_string,
filter_query,
start=None,
end=None):
# TODO resolve circular import
from onadata.libs.utils.csv_builder import CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(username, id_string, filter_query,
self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES,
self.BINARY_SELECT_MULTIPLES, start,
end, self.TRUNCATE_GROUP_TITLE)
csv_builder.export_to(path)
def test_windows_excel_compatible_csv_export(self):
"""
Test window excel compatible CSV export.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False,
win_excel_utf8=True)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name)
csv_reader = csv.reader(csv_file)
header = csv_reader.next()
self.assertEqual(len(header),
17 + len(AbstractDataFrameBuilder.ADDITIONAL_COLUMNS))
expected_header = [
'\xef\xbb\xbfname', '\xef\xbb\xbfage', '\xef\xbb\xbfhas_kids',
'\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age',
'\xef\xbb\xbfkids_name', '\xef\xbb\xbfkids_age', '\xef\xbb\xbfgps',
'\xef\xbb\xbf_gps_latitude', '\xef\xbb\xbf_gps_longitude',
'\xef\xbb\xbf_gps_altitude', '\xef\xbb\xbf_gps_precision',
'\xef\xbb\xbfweb_browsers/firefox',
'\xef\xbb\xbfweb_browsers/chrome', '\xef\xbb\xbfweb_browsers/ie',
'\xef\xbb\xbfweb_browsers/safari', '\xef\xbb\xbfinstanceID',
'\xef\xbb\xbf_id', '\xef\xbb\xbf_uuid',
'\xef\xbb\xbf_submission_time', '\xef\xbb\xbf_tags',
'\xef\xbb\xbf_notes', '\xef\xbb\xbf_version',
'\xef\xbb\xbf_duration', '\xef\xbb\xbf_submitted_by',
'\xef\xbb\xbf_total_media', '\xef\xbb\xbf_media_count',
'\xef\xbb\xbf_media_all_received'
]
self.assertEqual(expected_header, header)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_export_remove_group_name(self):
"""
Test CSV export with remove_group_name option.
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_images=False)
# pylint: disable=protected-access
record_count = csv_df_builder._query_data(count=True)
self.assertEqual(record_count, 7)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received'
]
self.assertEqual(expected_header, header)
rows = []
for row in csv_reader:
rows.append(row)
self.assertEqual(len(rows), 7)
self.assertEqual(rows[4][5], NA_REP)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
def test_csv_dataframe_export_to(self):
self._publish_nested_repeats_form()
self._submit_fixture_instance(
"nested_repeats", "01", submission_time=self._submission_time)
self._submit_fixture_instance(
"nested_repeats", "02", submission_time=self._submission_time)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_fixture_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"fixtures", "nested_repeats", "nested_repeats.csv"
)
temp_file.close()
fixture, output = '', ''
with open(csv_fixture_path) as f:
fixture = f.read()
with open(temp_file.name) as f:
output = f.read()
os.unlink(temp_file.name)
self.assertEqual(fixture, output)
def test_csv_export_extra_columns(self):
"""
Test CSV export EXTRA_COLUMNS
"""
self._publish_single_level_repeat_form()
# submit 7 instances
for _ in range(4):
self._submit_fixture_instance("new_repeats", "01")
self._submit_fixture_instance("new_repeats", "02")
for _ in range(2):
self._submit_fixture_instance("new_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
expected_header = [
'name', 'age', 'has_kids', 'kids_name', 'kids_age', 'kids_name',
'kids_age', 'gps', '_gps_latitude', '_gps_longitude',
'_gps_altitude', '_gps_precision', 'web_browsers/firefox',
'web_browsers/chrome', 'web_browsers/ie', 'web_browsers/safari',
'instanceID', '_id', '_uuid', '_submission_time', '_tags',
'_notes', '_version', '_duration', '_submitted_by', '_total_media',
'_media_count', '_media_all_received', '_xform_id'
]
self.assertEqual(expected_header, header)
# close and delete file
csv_file.close()
os.unlink(temp_file.name)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
remove_group_name=True,
include_labels=True)
temp_file = NamedTemporaryFile(suffix=".csv", delete=False)
csv_df_builder.export_to(temp_file.name)
csv_file = open(temp_file.name, 'r')
csv_reader = csv.reader(csv_file)
header = next(csv_reader)
self.assertEqual(len(header), 17 + len(csv_df_builder.extra_columns))
self.assertEqual(expected_header, header)
csv_file.close()
os.unlink(temp_file.name)
def test_split_select_multiples_within_repeats(self):
self.maxDiff = None
record = {
'name': 'Tom',
'age': 23,
'browser_use': [
{
'browser_use/year': '2010',
'browser_use/browsers': 'firefox safari'
},
{
'browser_use/year': '2011',
'browser_use/browsers': 'firefox chrome'
}
]
}
expected_result = {
'name': 'Tom',
'age': 23,
'browser_use': [
{
'browser_use/year': '2010',
'browser_use/browsers/firefox': True,
'browser_use/browsers/safari': True,
'browser_use/browsers/ie': False,
'browser_use/browsers/chrome': False
},
{
'browser_use/year': '2011',
'browser_use/browsers/firefox': True,
'browser_use/browsers/safari': False,
'browser_use/browsers/ie': False,
'browser_use/browsers/chrome': True
}
]
}
select_multiples = {
'browser_use/browsers': [
'browser_use/browsers/firefox',
'browser_use/browsers/safari',
'browser_use/browsers/ie',
'browser_use/browsers/chrome']}
result = CSVDataFrameBuilder._split_select_multiples(record,
select_multiples)
self.assertEqual(expected_result, result)
def test_split_select_multiples_within_repeats(self):
"""
Test select multiples choices are split within repeats in CSV exports.
"""
self.maxDiff = None
record = {
'name': 'Tom',
'age': 23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers': 'firefox safari'
}, {
'browser_use/year': '2011',
'browser_use/browsers': 'firefox chrome'
}]
} # yapf: disable
expected_result = {
'name': 'Tom',
'age': 23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers/firefox': True,
'browser_use/browsers/safari': True,
'browser_use/browsers/ie': False,
'browser_use/browsers/chrome': False
}, {
'browser_use/year': '2011',
'browser_use/browsers/firefox': True,
'browser_use/browsers/safari': False,
'browser_use/browsers/ie': False,
'browser_use/browsers/chrome': True
}]
} # yapf: disable
select_multiples = {
'browser_use/browsers': [
'browser_use/browsers/firefox', 'browser_use/browsers/safari',
'browser_use/browsers/ie', 'browser_use/browsers/chrome'
]
}
# pylint: disable=protected-access
result = CSVDataFrameBuilder._split_select_multiples(
record, select_multiples)
self.assertEqual(expected_result, result)
def test_split_select_multiples_within_repeats(self):
self.maxDiff = None
record = {
'name':
'Tom',
'age':
23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers': 'firefox safari'
}, {
'browser_use/year': '2011',
'browser_use/browsers': 'firefox chrome'
}]
}
expected_result = {
'name':
'Tom',
'age':
23,
'browser_use': [{
'browser_use/year': '2010',
'browser_use/browsers/firefox': True,
'browser_use/browsers/safari': True,
'browser_use/browsers/ie': False,
'browser_use/browsers/chrome': False
}, {
'browser_use/year': '2011',
'browser_use/browsers/firefox': True,
'browser_use/browsers/safari': False,
'browser_use/browsers/ie': False,
'browser_use/browsers/chrome': True
}]
}
select_multiples = {
'browser_use/browsers': [
'browser_use/browsers/firefox', 'browser_use/browsers/safari',
'browser_use/browsers/ie', 'browser_use/browsers/chrome'
]
}
result = CSVDataFrameBuilder._split_select_multiples(
record, select_multiples)
self.assertEqual(expected_result, result)
def test_split_select_multiples(self):
self._publish_nested_repeats_form()
self._submit_fixture_instance("nested_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
cursor = [k for k in csv_df_builder._query_data()]
record = cursor[0]
select_multiples = \
CSVDataFrameBuilder._collect_select_multiples(self.xform)
result = CSVDataFrameBuilder._split_select_multiples(
record, select_multiples)
expected_result = {
u'web_browsers/ie': True,
u'web_browsers/safari': True,
u'web_browsers/firefox': False,
u'web_browsers/chrome': False
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys()
if key in expected_result.keys()])
self.assertEqual(expected_result, result)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
binary_select_multiples=True)
result = csv_df_builder._split_select_multiples(
record, select_multiples)
expected_result = {
u'web_browsers/ie': 1,
u'web_browsers/safari': 1,
u'web_browsers/firefox': 0,
u'web_browsers/chrome': 0
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys()
if key in expected_result.keys()])
self.assertEqual(expected_result, result)
def test_split_select_multiples(self):
self._publish_nested_repeats_form()
dd = self.xform.data_dictionary()
self._submit_fixture_instance("nested_repeats", "01")
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
cursor = [k for k in csv_df_builder._query_data()]
record = cursor[0]
select_multiples = CSVDataFrameBuilder._collect_select_multiples(dd)
result = CSVDataFrameBuilder._split_select_multiples(record,
select_multiples)
expected_result = {
u'web_browsers/ie': True,
u'web_browsers/safari': True,
u'web_browsers/firefox': False,
u'web_browsers/chrome': False
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys() if key in
expected_result.keys()])
self.assertEqual(expected_result, result)
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
binary_select_multiples=True)
result = csv_df_builder._split_select_multiples(record,
select_multiples)
expected_result = {
u'web_browsers/ie': 1,
u'web_browsers/safari': 1,
u'web_browsers/firefox': 0,
u'web_browsers/chrome': 0
}
# build a new dictionary only composed of the keys we want to use in
# the comparison
result = dict([(key, result[key]) for key in result.keys() if key in
expected_result.keys()])
self.assertEqual(expected_result, result)
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False)
cursor = csv_df_builder._query_data()
return csv_df_builder._format_for_dataframe(cursor)
def _csv_data_for_dataframe(self):
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string)
cursor = csv_df_builder._query_data()
return csv_df_builder._format_for_dataframe(cursor)
def test_index_tag_replacement(self):
"""
Test that the default index tags are correctly replaced by an
underscore
"""
self._publish_xls_fixture_set_xform("groups_in_repeats")
self._submit_fixture_instance("groups_in_repeats", "01")
self.xform.get_keys()
csv_df_builder = CSVDataFrameBuilder(self.user.username,
self.xform.id_string,
include_images=False,
index_tags=('_', '_'))
cursor = csv_df_builder._query_data()
result = [d for d in csv_df_builder._format_for_dataframe(cursor)][0]
# remove dynamic fields
ignore_list = [
'_uuid', 'meta/instanceID', 'formhub/uuid', '_submission_time',
'_id', '_bamboo_dataset_id'
]
for item in ignore_list:
result.pop(item)
expected_result = {
u'_xform_id_string': u'groups_in_repeats',
u'_xform_id': self.xform.pk,
u'_status': u'submitted_via_web',
u'_tags': u'',
u'_notes': u'',
u'_version': self.xform.version,
u"_submitted_by": u'bob',
u'name': u'Abe',
u'age': 88,
u'has_children': u'1',
u'children_1_/childs_info/name': u'Cain',
u'children_2_/childs_info/name': u'Abel',
u'children_1_/childs_info/age': 56,
u'children_2_/childs_info/age': 48,
u'children_1_/immunization/immunization_received/polio_1': True,
u'children_1_/immunization/immunization_received/polio_2': False,
u'children_2_/immunization/immunization_received/polio_1': True,
u'children_2_/immunization/immunization_received/polio_2': True,
u'web_browsers/chrome': True,
u'web_browsers/firefox': False,
u'web_browsers/ie': False,
u'web_browsers/safari': False,
u'gps': u'-1.2626156 36.7923571 0.0 30.0',
u'_geolocation': [-1.2626156, 36.7923571],
u'_duration': '',
u'_edited': False,
u'_gps_latitude': u'-1.2626156',
u'_gps_longitude': u'36.7923571',
u'_gps_altitude': u'0.0',
u'_gps_precision': u'30.0',
u'_attachments': [],
u'_total_media': 0,
u'_media_count': 0,
u'_media_all_received': True
}
self.maxDiff = None
self.assertEqual(expected_result, result)
