def test_type_conversion(self):
submission_1 = {
"_id": 579827,
"geolocation": "-1.2625482 36.7924794 0.0 21.0",
"_bamboo_dataset_id": "",
"meta/instanceID": "uuid:2a8129f5-3091-44e1-a579-bed2b07a12cf",
"name": "Smith",
"formhub/uuid": "633ec390e024411ba5ce634db7807e62",
"_submission_time": "2013-07-03T08:25:30",
"age": "107",
"_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf",
"when": "2013-07-03",
"amount": "250.0",
"_geolocation": [
"-1.2625482",
"36.7924794"
],
"_xform_id_string": "test_data_types",
"_userform_id": "larryweya_test_data_types",
"_status": "submitted_via_web",
"precisely": "2013-07-03T15:24:00.000+03",
"really": "15:24:00.000+03"
}
submission_2 = {
"_id": 579828,
"_submission_time": "2013-07-03T08:26:10",
"_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5",
"_bamboo_dataset_id": "",
"_xform_id_string": "test_data_types",
"_userform_id": "larryweya_test_data_types",
"_status": "submitted_via_web",
"meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5",
"formhub/uuid": "633ec390e024411ba5ce634db7807e62",
"amount": "",
}
survey = create_survey_from_xls(viewer_fixture_path(
'test_data_types/test_data_types.xls'))
export_builder = ExportBuilder()
export_builder.set_survey(survey)
# format submission 1 for export
survey_name = survey.name
indices = {survey_name: 0}
data = dict_to_joined_export(submission_1, 1, indices, survey_name)
new_row = export_builder.pre_process_row(data[survey_name],
export_builder.sections[0])
self.assertIsInstance(new_row['age'], int)
self.assertIsInstance(new_row['when'], datetime.date)
self.assertIsInstance(new_row['amount'], float)
# check missing values dont break and empty values return blank strings
indices = {survey_name: 0}
data = dict_to_joined_export(submission_2, 1, indices, survey_name)
new_row = export_builder.pre_process_row(data[survey_name],
export_builder.sections[0])
self.assertIsInstance(new_row['amount'], basestring)
self.assertEqual(new_row['amount'], '')
def test_type_conversion(self):
submission_1 = {
"_id": 579827,
"geolocation": "-1.2625482 36.7924794 0.0 21.0",
"_bamboo_dataset_id": "",
"meta/instanceID": "uuid:2a8129f5-3091-44e1-a579-bed2b07a12cf",
"name": "Smith",
"formhub/uuid": "633ec390e024411ba5ce634db7807e62",
"_submission_time": "2013-07-03T08:25:30",
"age": "107",
"_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf",
"when": "2013-07-03",
"amount": "250.0",
"_geolocation": [
"-1.2625482",
"36.7924794"
],
"_xform_id_string": "test_data_types",
"_userform_id": "larryweya_test_data_types",
"_status": "submitted_via_web",
"precisely": "2013-07-03T15:24:00.000+03",
"really": "15:24:00.000+03"
}
submission_2 = {
"_id": 579828,
"_submission_time": "2013-07-03T08:26:10",
"_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5",
"_bamboo_dataset_id": "",
"_xform_id_string": "test_data_types",
"_userform_id": "larryweya_test_data_types",
"_status": "submitted_via_web",
"meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5",
"formhub/uuid": "633ec390e024411ba5ce634db7807e62",
"amount": "",
}
survey = create_survey_from_xls(viewer_fixture_path(
'test_data_types/test_data_types.xls'))
export_builder = ExportBuilder()
export_builder.set_survey(survey)
# format submission 1 for export
survey_name = survey.name
indices = {survey_name: 0}
data = dict_to_joined_export(submission_1, 1, indices, survey_name)
new_row = export_builder.pre_process_row(data[survey_name],
export_builder.sections[0])
self.assertIsInstance(new_row['age'], int)
self.assertIsInstance(new_row['when'], datetime.date)
self.assertIsInstance(new_row['amount'], float)
# check missing values dont break and empty values return blank strings
indices = {survey_name: 0}
data = dict_to_joined_export(submission_2, 1, indices, survey_name)
new_row = export_builder.pre_process_row(data[survey_name],
export_builder.sections[0])
self.assertIsInstance(new_row['amount'], basestring)
self.assertEqual(new_row['amount'], '')
def test_dict_to_joined_export_notes(self):
submission = {
"_id": 579828,
"_submission_time": "2013-07-03T08:26:10",
"_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5",
"_bamboo_dataset_id": "",
"_xform_id_string": "test_data_types",
"_userform_id": "larryweya_test_data_types",
"_status": "submitted_via_web",
"_notes": [
{
"note": "Note 1",
"date_created": "2013-07-03T08:26:10",
"id": 356,
"date_modified": "2013-07-03T08:26:10"
},
{
"note": "Note 2",
"date_created": "2013-07-03T08:34:40",
"id": 357,
"date_modified": "2013-07-03T08:34:40"
},
{
"note": "Note 3",
"date_created": "2013-07-03T08:56:14",
"id": 361,
"date_modified": "2013-07-03T08:56:14"
}
],
"meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5",
"formhub/uuid": "633ec390e024411ba5ce634db7807e62",
"amount": "",
}
survey_name = 'tutorial'
indices = {survey_name: 0}
data = dict_to_joined_export(submission, 1, indices, survey_name)
expected_data = {
'tutorial': {
'_id': 579828,
'_submission_time': '2013-07-03T08:26:10',
'_uuid': '5b4752eb-e13c-483e-87cb-e67ca6bb61e5',
'_bamboo_dataset_id': '',
'amount': '',
'_xform_id_string': 'test_data_types',
'_userform_id': 'larryweya_test_data_types',
'_status': 'submitted_via_web',
'_notes': 'Note 1\nNote 2\nNote 3',
'meta/instanceID': 'uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5',
'formhub/uuid': '633ec390e024411ba5ce634db7807e62'
}
}
self.assertEqual(sorted(data), sorted(expected_data))
def test_to_sav_export(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
temp_zip_file = NamedTemporaryFile(suffix='.zip')
filename = temp_zip_file.name
export_builder.to_zipped_sav(filename, self.data)
temp_zip_file.seek(0)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
zip_file.extractall(temp_dir)
zip_file.close()
temp_zip_file.close()
# generate data to compare with
index = 1
indices = {}
survey_name = survey.name
outputs = []
for d in self.data:
outputs.append(
dict_to_joined_export(d, index, indices, survey_name))
index += 1
# check that each file exists
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "{0}.sav".format(survey.name))))
def _test_sav_file(section):
with SavReader(
os.path.join(
temp_dir, "{0}.sav".format(section)),
returnHeader=True) as reader:
header = next(reader)
rows = [r for r in reader]
# open comparison file
with SavReader(_logger_fixture_path(
'spss', "{0}.sav".format(section)),
returnHeader=True) as fixture_reader:
fixture_header = next(fixture_reader)
self.assertEqual(header, fixture_header)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
for section in export_builder.sections:
section_name = section['name'].replace('/', '_')
_test_sav_file(section_name)
def test_to_sav_export(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
temp_zip_file = NamedTemporaryFile(suffix='.zip')
filename = temp_zip_file.name
export_builder.to_zipped_sav(filename, self.data)
temp_zip_file.seek(0)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
zip_file.extractall(temp_dir)
zip_file.close()
temp_zip_file.close()
# generate data to compare with
index = 1
indices = {}
survey_name = survey.name
outputs = []
for d in self.data:
outputs.append(
dict_to_joined_export(d, index, indices, survey_name))
index += 1
# check that each file exists
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "{0}.sav".format(survey.name))))
def _test_sav_file(section):
with SavReader(
os.path.join(
temp_dir, "{0}.sav".format(section)),
returnHeader=True) as reader:
header = next(reader)
rows = [r for r in reader]
# open comparison file
with SavReader(_logger_fixture_path(
'spss', "{0}.sav".format(section)),
returnHeader=True) as fixture_reader:
fixture_header = next(fixture_reader)
self.assertEqual(header, fixture_header)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
for section in export_builder.sections:
section_name = section['name'].replace('/', '_')
_test_sav_file(section_name)
def test_dict_to_joined_export_works(self):
data =\
{
'name': 'Abe',
'age': '35',
'_geolocation': [None, None],
'attachments': ['abcd.jpg', 'efgh.jpg'],
'children':
[
{
'children/name': 'Mike',
'children/age': '5',
'children/cartoons':
[
{
'children/cartoons/name': 'Tom & Jerry',
'children/cartoons/why': 'Tom is silly',
},
{
'children/cartoons/name': 'Flinstones',
'children/cartoons/why':
u"I like bamb bam\u0107",
}
]
},
{
'children/name': 'John',
'children/age': '2',
'children/cartoons': []
},
{
'children/name': 'Imora',
'children/age': '3',
'children/cartoons':
[
{
'children/cartoons/name': 'Shrek',
'children/cartoons/why': 'He\'s so funny'
},
{
'children/cartoons/name': 'Dexter\'s Lab',
'children/cartoons/why': 'He thinks hes smart',
'children/cartoons/characters':
[
{
'children/cartoons/characters/name':
'Dee Dee',
'children/cartoons/characters/good_or_'
'evil': 'good'
},
{
'children/cartoons/characters/name':
'Dexter',
'children/cartoons/characters/good_or_'
'evil': 'evil'
},
]
}
]
}
]
}
expected_output =\
{
'survey': {
'name': 'Abe',
'age': '35'
},
'children':
[
{
'children/name': 'Mike',
'children/age': '5',
'_index': 1,
'_parent_table_name': 'survey',
'_parent_index': 1
},
{
'children/name': 'John',
'children/age': '2',
'_index': 2,
'_parent_table_name': 'survey',
'_parent_index': 1
},
{
'children/name': 'Imora',
'children/age': '3',
'_index': 3,
'_parent_table_name': 'survey',
'_parent_index': 1
},
],
'children/cartoons':
[
{
'children/cartoons/name': 'Tom & Jerry',
'children/cartoons/why': 'Tom is silly',
'_index': 1,
'_parent_table_name': 'children',
'_parent_index': 1
},
{
'children/cartoons/name': 'Flinstones',
'children/cartoons/why': u"I like bamb bam\u0107",
'_index': 2,
'_parent_table_name': 'children',
'_parent_index': 1
},
{
'children/cartoons/name': 'Shrek',
'children/cartoons/why': 'He\'s so funny',
'_index': 3,
'_parent_table_name': 'children',
'_parent_index': 3
},
{
'children/cartoons/name': 'Dexter\'s Lab',
'children/cartoons/why': 'He thinks hes smart',
'_index': 4,
'_parent_table_name': 'children',
'_parent_index': 3
}
],
'children/cartoons/characters':
[
{
'children/cartoons/characters/name': 'Dee Dee',
'children/cartoons/characters/good_or_evil': 'good',
'_index': 1,
'_parent_table_name': 'children/cartoons',
'_parent_index': 4
},
{
'children/cartoons/characters/name': 'Dexter',
'children/cartoons/characters/good_or_evil': 'evil',
'_index': 2,
'_parent_table_name': 'children/cartoons',
'_parent_index': 4
}
]
}
survey_name = 'survey'
indices = {survey_name: 0}
output = dict_to_joined_export(data, 1, indices, survey_name)
self.assertEqual(output[survey_name], expected_output[survey_name])
# 1st level
self.assertEqual(len(output['children']), 3)
for child in enumerate(['Mike', 'John', 'Imora']):
index = child[0]
name = child[1]
self.assertEqual(
filter(
lambda x: x['children/name'] == name,
output['children'])[0],
expected_output['children'][index])
# 2nd level
self.assertEqual(len(output['children/cartoons']), 4)
for cartoon in enumerate(
['Tom & Jerry', 'Flinstones', 'Shrek', 'Dexter\'s Lab']):
index = cartoon[0]
name = cartoon[1]
self.assertEqual(
filter(
lambda x: x['children/cartoons/name'] == name,
output['children/cartoons'])[0],
expected_output['children/cartoons'][index])
# 3rd level
self.assertEqual(len(output['children/cartoons/characters']), 2)
for characters in enumerate(['Dee Dee', 'Dexter']):
index = characters[0]
name = characters[1]
self.assertEqual(
filter(
lambda x: x['children/cartoons/characters/name'] == name,
output['children/cartoons/characters'])[0],
expected_output['children/cartoons/characters'][index])
def test_zipped_csv_export_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
temp_zip_file = NamedTemporaryFile(suffix='.zip')
export_builder.to_zipped_csv(temp_zip_file.name, self.data)
temp_zip_file.seek(0)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
zip_file.extractall(temp_dir)
zip_file.close()
temp_zip_file.close()
# generate data to compare with
index = 1
indices = {}
survey_name = survey.name
outputs = []
for d in self.data:
outputs.append(
dict_to_joined_export(d, index, indices, survey_name))
index += 1
# check that each file exists
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "{0}.csv".format(survey.name))))
with open(os.path.join(temp_dir,
"{0}.csv".format(survey.name))) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'childrens_survey.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv")))
with open(os.path.join(temp_dir, "children.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path('csvs',
'children.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(os.path.join(temp_dir, "children_cartoons.csv")))
with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'children_cartoons.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children_cartoons_characters.csv")))
with open(os.path.join(
temp_dir, "children_cartoons_characters.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(
_logger_fixture_path(
'csvs',
'children_cartoons_characters.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
shutil.rmtree(temp_dir)
def test_zipped_csv_export_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
temp_zip_file = NamedTemporaryFile(suffix='.zip')
export_builder.to_zipped_csv(temp_zip_file.name, self.data)
temp_zip_file.seek(0)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
zip_file.extractall(temp_dir)
zip_file.close()
temp_zip_file.close()
# generate data to compare with
index = 1
indices = {}
survey_name = survey.name
outputs = []
for d in self.data:
outputs.append(
dict_to_joined_export(d, index, indices, survey_name))
index += 1
# check that each file exists
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "{0}.csv".format(survey.name))))
with open(
os.path.join(
temp_dir, "{0}.csv".format(survey.name))) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'childrens_survey.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children.csv")))
with open(os.path.join(temp_dir, "children.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'children.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children_cartoons.csv")))
with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'children_cartoons.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children_cartoons_characters.csv")))
with open(os.path.join(
temp_dir, "children_cartoons_characters.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs',
'children_cartoons_characters.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
shutil.rmtree(temp_dir)
