def test_type_conversion(self): submission_1 = { "_id": 579827, "geolocation": "-1.2625482 36.7924794 0.0 21.0", "_bamboo_dataset_id": "", "meta/instanceID": "uuid:2a8129f5-3091-44e1-a579-bed2b07a12cf", "name": "Smith", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "_submission_time": "2013-07-03T08:25:30", "age": "107", "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", "_geolocation": [ "-1.2625482", "36.7924794" ], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", "really": "15:24:00.000+03" } submission_2 = { "_id": 579828, "_submission_time": "2013-07-03T08:26:10", "_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "_bamboo_dataset_id": "", "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "amount": "", } survey = create_survey_from_xls(viewer_fixture_path( 'test_data_types/test_data_types.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} data = dict_to_joined_export(submission_1, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['age'], int) self.assertIsInstance(new_row['when'], datetime.date) self.assertIsInstance(new_row['amount'], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} data = dict_to_joined_export(submission_2, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['amount'], basestring) self.assertEqual(new_row['amount'], '')
def test_type_conversion(self): submission_1 = { "_id": 579827, "geolocation": "-1.2625482 36.7924794 0.0 21.0", "_bamboo_dataset_id": "", "meta/instanceID": "uuid:2a8129f5-3091-44e1-a579-bed2b07a12cf", "name": "Smith", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "_submission_time": "2013-07-03T08:25:30", "age": "107", "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", "_geolocation": [ "-1.2625482", "36.7924794" ], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", "really": "15:24:00.000+03" } submission_2 = { "_id": 579828, "_submission_time": "2013-07-03T08:26:10", "_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "_bamboo_dataset_id": "", "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "amount": "", } survey = create_survey_from_xls(viewer_fixture_path( 'test_data_types/test_data_types.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} data = dict_to_joined_export(submission_1, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['age'], int) self.assertIsInstance(new_row['when'], datetime.date) self.assertIsInstance(new_row['amount'], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} data = dict_to_joined_export(submission_2, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['amount'], basestring) self.assertEqual(new_row['amount'], '')
def test_dict_to_joined_export_notes(self): submission = { "_id": 579828, "_submission_time": "2013-07-03T08:26:10", "_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "_bamboo_dataset_id": "", "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "_notes": [ { "note": "Note 1", "date_created": "2013-07-03T08:26:10", "id": 356, "date_modified": "2013-07-03T08:26:10" }, { "note": "Note 2", "date_created": "2013-07-03T08:34:40", "id": 357, "date_modified": "2013-07-03T08:34:40" }, { "note": "Note 3", "date_created": "2013-07-03T08:56:14", "id": 361, "date_modified": "2013-07-03T08:56:14" } ], "meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "amount": "", } survey_name = 'tutorial' indices = {survey_name: 0} data = dict_to_joined_export(submission, 1, indices, survey_name) expected_data = { 'tutorial': { '_id': 579828, '_submission_time': '2013-07-03T08:26:10', '_uuid': '5b4752eb-e13c-483e-87cb-e67ca6bb61e5', '_bamboo_dataset_id': '', 'amount': '', '_xform_id_string': 'test_data_types', '_userform_id': 'larryweya_test_data_types', '_status': 'submitted_via_web', '_notes': 'Note 1\nNote 2\nNote 3', 'meta/instanceID': 'uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5', 'formhub/uuid': '633ec390e024411ba5ce634db7807e62' } } self.assertEqual(sorted(data), sorted(expected_data))
def test_to_sav_export(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.sav".format(survey.name)))) def _test_sav_file(section): with SavReader( os.path.join( temp_dir, "{0}.sav".format(section)), returnHeader=True) as reader: header = next(reader) rows = [r for r in reader] # open comparison file with SavReader(_logger_fixture_path( 'spss', "{0}.sav".format(section)), returnHeader=True) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) for section in export_builder.sections: section_name = section['name'].replace('/', '_') _test_sav_file(section_name)
def test_to_sav_export(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.sav".format(survey.name)))) def _test_sav_file(section): with SavReader( os.path.join( temp_dir, "{0}.sav".format(section)), returnHeader=True) as reader: header = next(reader) rows = [r for r in reader] # open comparison file with SavReader(_logger_fixture_path( 'spss', "{0}.sav".format(section)), returnHeader=True) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) for section in export_builder.sections: section_name = section['name'].replace('/', '_') _test_sav_file(section_name)
def test_dict_to_joined_export_works(self): data =\ { 'name': 'Abe', 'age': '35', '_geolocation': [None, None], 'attachments': ['abcd.jpg', 'efgh.jpg'], 'children': [ { 'children/name': 'Mike', 'children/age': '5', 'children/cartoons': [ { 'children/cartoons/name': 'Tom & Jerry', 'children/cartoons/why': 'Tom is silly', }, { 'children/cartoons/name': 'Flinstones', 'children/cartoons/why': u"I like bamb bam\u0107", } ] }, { 'children/name': 'John', 'children/age': '2', 'children/cartoons': [] }, { 'children/name': 'Imora', 'children/age': '3', 'children/cartoons': [ { 'children/cartoons/name': 'Shrek', 'children/cartoons/why': 'He\'s so funny' }, { 'children/cartoons/name': 'Dexter\'s Lab', 'children/cartoons/why': 'He thinks hes smart', 'children/cartoons/characters': [ { 'children/cartoons/characters/name': 'Dee Dee', 'children/cartoons/characters/good_or_' 'evil': 'good' }, { 'children/cartoons/characters/name': 'Dexter', 'children/cartoons/characters/good_or_' 'evil': 'evil' }, ] } ] } ] } expected_output =\ { 'survey': { 'name': 'Abe', 'age': '35' }, 'children': [ { 'children/name': 'Mike', 'children/age': '5', '_index': 1, '_parent_table_name': 'survey', '_parent_index': 1 }, { 'children/name': 'John', 'children/age': '2', '_index': 2, '_parent_table_name': 'survey', '_parent_index': 1 }, { 'children/name': 'Imora', 'children/age': '3', '_index': 3, '_parent_table_name': 'survey', '_parent_index': 1 }, ], 'children/cartoons': [ { 'children/cartoons/name': 'Tom & Jerry', 'children/cartoons/why': 'Tom is silly', '_index': 1, '_parent_table_name': 'children', '_parent_index': 1 }, { 'children/cartoons/name': 'Flinstones', 'children/cartoons/why': u"I like bamb bam\u0107", '_index': 2, '_parent_table_name': 'children', '_parent_index': 1 }, { 'children/cartoons/name': 'Shrek', 'children/cartoons/why': 'He\'s so funny', '_index': 3, '_parent_table_name': 'children', '_parent_index': 3 }, { 'children/cartoons/name': 'Dexter\'s Lab', 'children/cartoons/why': 'He thinks hes smart', '_index': 4, '_parent_table_name': 'children', '_parent_index': 3 } ], 'children/cartoons/characters': [ { 'children/cartoons/characters/name': 'Dee Dee', 'children/cartoons/characters/good_or_evil': 'good', '_index': 1, '_parent_table_name': 'children/cartoons', '_parent_index': 4 }, { 'children/cartoons/characters/name': 'Dexter', 'children/cartoons/characters/good_or_evil': 'evil', '_index': 2, '_parent_table_name': 'children/cartoons', '_parent_index': 4 } ] } survey_name = 'survey' indices = {survey_name: 0} output = dict_to_joined_export(data, 1, indices, survey_name) self.assertEqual(output[survey_name], expected_output[survey_name]) # 1st level self.assertEqual(len(output['children']), 3) for child in enumerate(['Mike', 'John', 'Imora']): index = child[0] name = child[1] self.assertEqual( filter( lambda x: x['children/name'] == name, output['children'])[0], expected_output['children'][index]) # 2nd level self.assertEqual(len(output['children/cartoons']), 4) for cartoon in enumerate( ['Tom & Jerry', 'Flinstones', 'Shrek', 'Dexter\'s Lab']): index = cartoon[0] name = cartoon[1] self.assertEqual( filter( lambda x: x['children/cartoons/name'] == name, output['children/cartoons'])[0], expected_output['children/cartoons'][index]) # 3rd level self.assertEqual(len(output['children/cartoons/characters']), 2) for characters in enumerate(['Dee Dee', 'Dexter']): index = characters[0] name = characters[1] self.assertEqual( filter( lambda x: x['children/cartoons/characters/name'] == name, output['children/cartoons/characters'])[0], expected_output['children/cartoons/characters'][index])
def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.csv".format(survey.name)))) with open(os.path.join(temp_dir, "{0}.csv".format(survey.name))) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'childrens_survey.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv"))) with open(os.path.join(temp_dir, "children.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path('csvs', 'children.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists(os.path.join(temp_dir, "children_cartoons.csv"))) with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons_characters.csv"))) with open(os.path.join( temp_dir, "children_cartoons_characters.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open( _logger_fixture_path( 'csvs', 'children_cartoons_characters.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir)
def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.csv".format(survey.name)))) with open( os.path.join( temp_dir, "{0}.csv".format(survey.name))) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'childrens_survey.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children.csv"))) with open(os.path.join(temp_dir, "children.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons.csv"))) with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons_characters.csv"))) with open(os.path.join( temp_dir, "children_cartoons_characters.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons_characters.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir)