def test_to_xls_export_respects_custom_field_delimiter(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = ExportBuilder.GROUP_DELIMITER_DOT export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check header columns main_sheet = wb.get_sheet_by_name('childrens_survey') expected_column_headers = [ u'name', u'age', u'geo.geolocation', u'geo._geolocation_latitude', u'geo._geolocation_longitude', u'geo._geolocation_altitude', u'geo._geolocation_precision', u'tel.tel.office', u'tel.tel.mobile', u'_id', u'meta.instanceID', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version' ] column_headers = [c[0].value for c in main_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close()
def test_generation_of_multi_selects_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_select_multiples =\ { 'children': { 'children/fav_colors': [ 'children/fav_colors/red', 'children/fav_colors/blue', 'children/fav_colors/pink' ], 'children/ice.creams': [ 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', 'children/ice.creams/chocolate' ] } } select_multiples = export_builder.select_multiples self.assertTrue('children' in select_multiples) self.assertTrue('children/fav_colors' in select_multiples['children']) self.assertTrue('children/ice.creams' in select_multiples['children']) self.assertEqual( sorted(select_multiples['children']['children/fav_colors']), sorted( expected_select_multiples['children']['children/fav_colors'])) self.assertEqual( sorted(select_multiples['children']['children/ice.creams']), sorted( expected_select_multiples['children']['children/ice.creams']))
def test_generation_of_multi_selects_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_select_multiples =\ { 'children': { 'children/fav_colors': [ 'children/fav_colors/red', 'children/fav_colors/blue', 'children/fav_colors/pink' ], 'children/ice.creams': [ 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', 'children/ice.creams/chocolate' ] } } select_multiples = export_builder.select_multiples self.assertTrue('children' in select_multiples) self.assertTrue('children/fav_colors' in select_multiples['children']) self.assertTrue('children/ice.creams' in select_multiples['children']) self.assertEqual( sorted(select_multiples['children']['children/fav_colors']), sorted( expected_select_multiples['children']['children/fav_colors'])) self.assertEqual( sorted(select_multiples['children']['children/ice.creams']), sorted( expected_select_multiples['children']['children/ice.creams']))
def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): survey = create_survey_from_xls( _logger_fixture_path('childrens_survey_with_a_very_long_name.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xlsx') filename = xls_file.name export_builder.to_xls_export(filename, self.long_survey_data) xls_file.seek(0) wb = load_workbook(filename) # get the children's sheet ws1 = wb.get_sheet_by_name('childrens_survey_with_a_very_l1') # parent_table is in cell K2 parent_table_name = ws1.cell('K2').value expected_parent_table_name = 'childrens_survey_with_a_very_lo' self.assertEqual(parent_table_name, expected_parent_table_name) # get cartoons sheet ws2 = wb.get_sheet_by_name('childrens_survey_with_a_very_l2') parent_table_name = ws2.cell('G2').value expected_parent_table_name = 'childrens_survey_with_a_very_l1' self.assertEqual(parent_table_name, expected_parent_table_name) xls_file.close()
def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): survey = create_survey_from_xls(_logger_fixture_path( 'childrens_survey_with_a_very_long_name.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xlsx') filename = xls_file.name export_builder.to_xls_export(filename, self.long_survey_data) xls_file.seek(0) wb = load_workbook(filename) # get the children's sheet ws1 = wb.get_sheet_by_name('childrens_survey_with_a_very_l1') # parent_table is in cell K2 parent_table_name = ws1.cell('K2').value expected_parent_table_name = 'childrens_survey_with_a_very_lo' self.assertEqual(parent_table_name, expected_parent_table_name) # get cartoons sheet ws2 = wb.get_sheet_by_name('childrens_survey_with_a_very_l2') parent_table_name = ws2.cell('G2').value expected_parent_table_name = 'childrens_survey_with_a_very_l1' self.assertEqual(parent_table_name, expected_parent_table_name) xls_file.close()
def test_type_conversion(self): submission_1 = { "_id": 579827, "geolocation": "-1.2625482 36.7924794 0.0 21.0", "_bamboo_dataset_id": "", "meta/instanceID": "uuid:2a8129f5-3091-44e1-a579-bed2b07a12cf", "name": "Smith", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "_submission_time": "2013-07-03T08:25:30", "age": "107", "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", "_geolocation": [ "-1.2625482", "36.7924794" ], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", "really": "15:24:00.000+03" } submission_2 = { "_id": 579828, "_submission_time": "2013-07-03T08:26:10", "_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "_bamboo_dataset_id": "", "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "amount": "", } survey = create_survey_from_xls(viewer_fixture_path( 'test_data_types/test_data_types.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} data = dict_to_joined_export(submission_1, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['age'], int) self.assertIsInstance(new_row['when'], datetime.date) self.assertIsInstance(new_row['amount'], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} data = dict_to_joined_export(submission_2, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['amount'], basestring) self.assertEqual(new_row['amount'], '')
def test_type_conversion(self): submission_1 = { "_id": 579827, "geolocation": "-1.2625482 36.7924794 0.0 21.0", "_bamboo_dataset_id": "", "meta/instanceID": "uuid:2a8129f5-3091-44e1-a579-bed2b07a12cf", "name": "Smith", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "_submission_time": "2013-07-03T08:25:30", "age": "107", "_uuid": "2a8129f5-3091-44e1-a579-bed2b07a12cf", "when": "2013-07-03", "amount": "250.0", "_geolocation": ["-1.2625482", "36.7924794"], "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "precisely": "2013-07-03T15:24:00.000+03", "really": "15:24:00.000+03" } submission_2 = { "_id": 579828, "_submission_time": "2013-07-03T08:26:10", "_uuid": "5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "_bamboo_dataset_id": "", "_xform_id_string": "test_data_types", "_userform_id": "larryweya_test_data_types", "_status": "submitted_via_web", "meta/instanceID": "uuid:5b4752eb-e13c-483e-87cb-e67ca6bb61e5", "formhub/uuid": "633ec390e024411ba5ce634db7807e62", "amount": "", } survey = create_survey_from_xls( viewer_fixture_path('test_data_types/test_data_types.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export survey_name = survey.name indices = {survey_name: 0} data = dict_to_joined_export(submission_1, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['age'], int) self.assertIsInstance(new_row['when'], datetime.date) self.assertIsInstance(new_row['amount'], float) # check missing values dont break and empty values return blank strings indices = {survey_name: 0} data = dict_to_joined_export(submission_2, 1, indices, survey_name) new_row = export_builder.pre_process_row(data[survey_name], export_builder.sections[0]) self.assertIsInstance(new_row['amount'], basestring) self.assertEqual(new_row['amount'], '')
def test_xls_convert_dates_before_1900(self): survey = create_survey_from_xls( viewer_fixture_path('test_data_types/test_data_types.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) data = [{ 'name': 'Abe', 'when': '1899-07-03', }] # create export file temp_xls_file = NamedTemporaryFile(suffix='.xlsx') export_builder.to_xls_export(temp_xls_file.name, data) temp_xls_file.close()
def test_zipped_csv_export_works_with_unicode(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ survey = create_survey_from_xls(_logger_fixture_path( 'childrens_survey_unicode.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "children.info.csv"))) # check file's contents with open(os.path.join(temp_dir, "children.info.csv")) as csv_file: reader = csv.reader(csv_file) expected_headers = ['children.info/name.first', 'children.info/age', 'children.info/fav_colors', u'children.info/fav_colors/red\u2019s', u'children.info/fav_colors/blue\u2019s', u'children.info/fav_colors/pink\u2019s', 'children.info/ice_creams', 'children.info/ice_creams/vanilla', 'children.info/ice_creams/strawberry', 'children.info/ice_creams/chocolate', '_id', '_uuid', '_submission_time', '_index', '_parent_table_name', '_parent_index', u'_tags', '_notes', '_version'] rows = [row for row in reader] actual_headers = [h.decode('utf-8') for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) self.assertEqual( data[u'children.info/fav_colors/red\u2019s'.encode('utf-8')], 'True') self.assertEqual( data[u'children.info/fav_colors/blue\u2019s'.encode('utf-8')], 'True') self.assertEqual( data[u'children.info/fav_colors/pink\u2019s'.encode('utf-8')], 'False') # check that red and blue are set to true shutil.rmtree(temp_dir)
def test_build_sections_from_survey(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) # test that we generate the proper sections expected_sections = [ survey.name, 'children', 'children/cartoons', 'children/cartoons/characters' ] self.assertEqual(expected_sections, [s['name'] for s in export_builder.sections]) # main section should have split geolocations expected_element_names = [ 'name', 'age', 'geo/geolocation', 'geo/_geolocation_longitude', 'geo/_geolocation_latitude', 'geo/_geolocation_altitude', 'geo/_geolocation_precision', 'tel/tel.office', 'tel/tel.mobile', 'meta/instanceID' ] section = export_builder.section_by_name(survey.name) element_names = [element['xpath'] for element in section['elements']] # fav_colors should have its choices split self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ 'children/name', 'children/age', 'children/fav_colors', 'children/fav_colors/red', 'children/fav_colors/blue', 'children/fav_colors/pink', 'children/ice.creams', 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', 'children/ice.creams/chocolate' ] section = export_builder.section_by_name('children') element_names = [element['xpath'] for element in section['elements']] self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ 'children/cartoons/name', 'children/cartoons/why' ] section = export_builder.section_by_name('children/cartoons') element_names = [element['xpath'] for element in section['elements']] self.assertEqual(sorted(expected_element_names), sorted(element_names)) expected_element_names = [ 'children/cartoons/characters/name', 'children/cartoons/characters/good_or_evil' ] section = \ export_builder.section_by_name('children/cartoons/characters') element_names = [element['xpath'] for element in section['elements']] self.assertEqual(sorted(expected_element_names), sorted(element_names))
def test_build_sections_from_survey(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) # test that we generate the proper sections expected_sections = [ survey.name, 'children', 'children/cartoons', 'children/cartoons/characters'] self.assertEqual( expected_sections, [s['name'] for s in export_builder.sections]) # main section should have split geolocations expected_element_names = [ 'name', 'age', 'geo/geolocation', 'geo/_geolocation_longitude', 'geo/_geolocation_latitude', 'geo/_geolocation_altitude', 'geo/_geolocation_precision', 'tel/tel.office', 'tel/tel.mobile', 'meta/instanceID'] section = export_builder.section_by_name(survey.name) element_names = [element['xpath'] for element in section['elements']] # fav_colors should have its choices split self.assertEqual( sorted(expected_element_names), sorted(element_names)) expected_element_names = [ 'children/name', 'children/age', 'children/fav_colors', 'children/fav_colors/red', 'children/fav_colors/blue', 'children/fav_colors/pink', 'children/ice.creams', 'children/ice.creams/vanilla', 'children/ice.creams/strawberry', 'children/ice.creams/chocolate'] section = export_builder.section_by_name('children') element_names = [element['xpath'] for element in section['elements']] self.assertEqual( sorted(expected_element_names), sorted(element_names)) expected_element_names = [ 'children/cartoons/name', 'children/cartoons/why'] section = export_builder.section_by_name('children/cartoons') element_names = [element['xpath'] for element in section['elements']] self.assertEqual( sorted(expected_element_names), sorted(element_names)) expected_element_names = [ 'children/cartoons/characters/name', 'children/cartoons/characters/good_or_evil'] section = \ export_builder.section_by_name('children/cartoons/characters') element_names = [element['xpath'] for element in section['elements']] self.assertEqual( sorted(expected_element_names), sorted(element_names))
def test_to_sav_export(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.sav".format(survey.name)))) def _test_sav_file(section): with SavReader( os.path.join( temp_dir, "{0}.sav".format(section)), returnHeader=True) as reader: header = next(reader) rows = [r for r in reader] # open comparison file with SavReader(_logger_fixture_path( 'spss', "{0}.sav".format(section)), returnHeader=True) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) for section in export_builder.sections: section_name = section['name'].replace('/', '_') _test_sav_file(section_name)
def test_zipped_csv_export_works_with_unicode(self): """ cvs writer doesnt handle unicode we we have to encode to ascii """ survey = create_survey_from_xls( _logger_fixture_path('childrens_survey_unicode.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data_utf8) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # check that the children's file (which has the unicode header) exists self.assertTrue( os.path.exists(os.path.join(temp_dir, "children.info.csv"))) # check file's contents with open(os.path.join(temp_dir, "children.info.csv")) as csv_file: reader = csv.reader(csv_file) expected_headers = [ 'children.info/name.first', 'children.info/age', 'children.info/fav_colors', u'children.info/fav_colors/red\u2019s', u'children.info/fav_colors/blue\u2019s', u'children.info/fav_colors/pink\u2019s', 'children.info/ice_creams', 'children.info/ice_creams/vanilla', 'children.info/ice_creams/strawberry', 'children.info/ice_creams/chocolate', '_id', '_uuid', '_submission_time', '_index', '_parent_table_name', '_parent_index', u'_tags', '_notes', '_version' ] rows = [row for row in reader] actual_headers = [h.decode('utf-8') for h in rows[0]] self.assertEqual(sorted(actual_headers), sorted(expected_headers)) data = dict(zip(rows[0], rows[1])) self.assertEqual( data[u'children.info/fav_colors/red\u2019s'.encode('utf-8')], 'True') self.assertEqual( data[u'children.info/fav_colors/blue\u2019s'.encode('utf-8')], 'True') self.assertEqual( data[u'children.info/fav_colors/pink\u2019s'.encode('utf-8')], 'False') # check that red and blue are set to true shutil.rmtree(temp_dir)
def test_xls_convert_dates_before_1900(self): survey = create_survey_from_xls(viewer_fixture_path( 'test_data_types/test_data_types.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) data = [ { 'name': 'Abe', 'when': '1899-07-03', } ] # create export file temp_xls_file = NamedTemporaryFile(suffix='.xlsx') export_builder.to_xls_export(temp_xls_file.name, data) temp_xls_file.close()
def test_xls_export_works_with_unicode(self): survey = create_survey_from_xls( _logger_fixture_path('childrens_survey_unicode.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) temp_xls_file = NamedTemporaryFile(suffix='.xlsx') export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\u2019s and blue\u2019s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb.get_sheet_by_name("children.info") data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) self.assertTrue(data[u'children.info/fav_colors/red\u2019s']) self.assertTrue(data[u'children.info/fav_colors/blue\u2019s']) self.assertFalse(data[u'children.info/fav_colors/pink\u2019s']) temp_xls_file.close()
def test_xls_export_works_with_unicode(self): survey = create_survey_from_xls(_logger_fixture_path( 'childrens_survey_unicode.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) temp_xls_file = NamedTemporaryFile(suffix='.xlsx') export_builder.to_xls_export(temp_xls_file.name, self.data_utf8) temp_xls_file.seek(0) # check that values for red\u2019s and blue\u2019s are set to true wb = load_workbook(temp_xls_file.name) children_sheet = wb.get_sheet_by_name("children.info") data = dict([(r[0].value, r[1].value) for r in children_sheet.columns]) self.assertTrue(data[u'children.info/fav_colors/red\u2019s']) self.assertTrue(data[u'children.info/fav_colors/blue\u2019s']) self.assertFalse(data[u'children.info/fav_colors/pink\u2019s']) temp_xls_file.close()
def test_to_sav_export(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') filename = temp_zip_file.name export_builder.to_zipped_sav(filename, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.sav".format(survey.name)))) def _test_sav_file(section): with SavReader(os.path.join(temp_dir, "{0}.sav".format(section)), returnHeader=True) as reader: header = next(reader) rows = [r for r in reader] # open comparison file with SavReader(_logger_fixture_path('spss', "{0}.sav".format(section)), returnHeader=True) as fixture_reader: fixture_header = next(fixture_reader) self.assertEqual(header, fixture_header) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) for section in export_builder.sections: section_name = section['name'].replace('/', '_') _test_sav_file(section_name)
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields =\ { 'childrens_survey': { 'tel/tel.office': 'tel/{0}'.format( _encode_for_mongo('tel.office')), 'tel/tel.mobile': 'tel/{0}'.format( _encode_for_mongo('tel.mobile')), } } encoded_fields = export_builder.encoded_fields self.assertTrue('childrens_survey' in encoded_fields) self.assertEqual(encoded_fields['childrens_survey'], expected_encoded_fields['childrens_survey'])
def test_to_xls_export_generates_valid_sheet_names(self): survey = create_survey_from_xls(_logger_fixture_path( 'childrens_survey_with_a_very_long_name.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets expected_sheet_names = ['childrens_survey_with_a_very_lo', 'childrens_survey_with_a_very_l1', 'childrens_survey_with_a_very_l2', 'childrens_survey_with_a_very_l3'] self.assertEqual(wb.get_sheet_names(), expected_sheet_names) xls_file.close()
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields =\ { 'childrens_survey': { 'tel/tel.office': 'tel/{0}'.format( _encode_for_mongo('tel.office')), 'tel/tel.mobile': 'tel/{0}'.format( _encode_for_mongo('tel.mobile')), } } encoded_fields = export_builder.encoded_fields self.assertTrue('childrens_survey' in encoded_fields) self.assertEqual( encoded_fields['childrens_survey'], expected_encoded_fields['childrens_survey'])
def test_delimiter_replacement_works_existing_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) expected_sections =\ [ { 'name': 'children', 'elements': [ { 'title': 'children.name', 'xpath': 'children/name' } ] } ] children_section = export_builder.section_by_name('children') self.assertEqual(children_section['elements'][0]['title'], expected_sections[0]['elements'][0]['title'])
def test_delimiter_replacement_works_generated_multi_select_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) expected_section =\ { 'name': 'children', 'elements': [ { 'title': 'children.fav_colors.red', 'xpath': 'children/fav_colors/red' } ] } childrens_section = export_builder.section_by_name('children') match = filter(lambda x: expected_section['elements'][0]['xpath'] == x['xpath'], childrens_section['elements'])[0] self.assertEqual( expected_section['elements'][0]['title'], match['title'])
def test_to_xls_export_generates_valid_sheet_names(self): survey = create_survey_from_xls( _logger_fixture_path('childrens_survey_with_a_very_long_name.xls')) export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets expected_sheet_names = [ 'childrens_survey_with_a_very_lo', 'childrens_survey_with_a_very_l1', 'childrens_survey_with_a_very_l2', 'childrens_survey_with_a_very_l3' ] self.assertEqual(wb.get_sheet_names(), expected_sheet_names) xls_file.close()
def test_delimiter_replacement_works_generated_multi_select_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) expected_section =\ { 'name': 'children', 'elements': [ { 'title': 'children.fav_colors.red', 'xpath': 'children/fav_colors/red' } ] } childrens_section = export_builder.section_by_name('children') match = filter( lambda x: expected_section['elements'][0]['xpath'] == x['xpath'], childrens_section['elements'])[0] self.assertEqual(expected_section['elements'][0]['title'], match['title'])
def test_delimiter_replacement_works_for_generated_gps_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) expected_section = \ { 'name': 'childrens_survey', 'elements': [ { 'title': 'geo._geolocation_latitude', 'xpath': 'geo/_geolocation_latitude' } ] } main_section = export_builder.section_by_name('childrens_survey') match = filter( lambda x: (expected_section['elements'][0]['xpath'] == x['xpath']), main_section['elements'])[0] self.assertEqual( expected_section['elements'][0]['title'], match['title'])
def test_delimiter_replacement_works_for_generated_gps_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) expected_section = \ { 'name': 'childrens_survey', 'elements': [ { 'title': 'geo._geolocation_latitude', 'xpath': 'geo/_geolocation_latitude' } ] } main_section = export_builder.section_by_name('childrens_survey') match = filter( lambda x: (expected_section['elements'][0]['xpath'] == x['xpath']), main_section['elements'])[0] self.assertEqual(expected_section['elements'][0]['title'], match['title'])
def test_generation_of_gps_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_gps_fields =\ { 'childrens_survey': { 'geo/geolocation': [ 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', 'geo/_geolocation_altitude', 'geo/_geolocation_precision' ] } } gps_fields = export_builder.gps_fields self.assertTrue('childrens_survey' in gps_fields) self.assertEqual(sorted(gps_fields['childrens_survey']), sorted(expected_gps_fields['childrens_survey']))
def test_delimiter_replacement_works_existing_fields(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = "." export_builder.set_survey(survey) expected_sections =\ [ { 'name': 'children', 'elements': [ { 'title': 'children.name', 'xpath': 'children/name' } ] } ] children_section = export_builder.section_by_name('children') self.assertEqual( children_section['elements'][0]['title'], expected_sections[0]['elements'][0]['title'])
def test_generation_of_gps_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_gps_fields =\ { 'childrens_survey': { 'geo/geolocation': [ 'geo/_geolocation_latitude', 'geo/_geolocation_longitude', 'geo/_geolocation_altitude', 'geo/_geolocation_precision' ] } } gps_fields = export_builder.gps_fields self.assertTrue('childrens_survey' in gps_fields) self.assertEqual( sorted(gps_fields['childrens_survey']), sorted(expected_gps_fields['childrens_survey']))
def test_to_xls_export_respects_custom_field_delimiter(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = ExportBuilder.GROUP_DELIMITER_DOT export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check header columns main_sheet = wb.get_sheet_by_name('childrens_survey') expected_column_headers = [ u'name', u'age', u'geo.geolocation', u'geo._geolocation_latitude', u'geo._geolocation_longitude', u'geo._geolocation_altitude', u'geo._geolocation_precision', u'tel.tel.office', u'tel.tel.mobile', u'_id', u'meta.instanceID', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version'] column_headers = [c[0].value for c in main_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close()
def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets expected_sheet_names = ['childrens_survey', 'children', 'children_cartoons', 'children_cartoons_characters'] self.assertEqual(wb.get_sheet_names(), expected_sheet_names) # check header columns main_sheet = wb.get_sheet_by_name('childrens_survey') expected_column_headers = [ u'name', u'age', u'geo/geolocation', u'geo/_geolocation_latitude', u'geo/_geolocation_longitude', u'geo/_geolocation_altitude', u'geo/_geolocation_precision', u'tel/tel.office', u'tel/tel.mobile', u'_id', u'meta/instanceID', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version'] column_headers = [c[0].value for c in main_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) childrens_sheet = wb.get_sheet_by_name('children') expected_column_headers = [ u'children/name', u'children/age', u'children/fav_colors', u'children/fav_colors/red', u'children/fav_colors/blue', u'children/fav_colors/pink', u'children/ice.creams', u'children/ice.creams/vanilla', u'children/ice.creams/strawberry', u'children/ice.creams/chocolate', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version'] column_headers = [c[0].value for c in childrens_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) cartoons_sheet = wb.get_sheet_by_name('children_cartoons') expected_column_headers = [ u'children/cartoons/name', u'children/cartoons/why', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version'] column_headers = [c[0].value for c in cartoons_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) characters_sheet = wb.get_sheet_by_name('children_cartoons_characters') expected_column_headers = [ u'children/cartoons/characters/name', u'children/cartoons/characters/good_or_evil', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version'] column_headers = [c[0].value for c in characters_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close()
def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.csv".format(survey.name)))) with open( os.path.join( temp_dir, "{0}.csv".format(survey.name))) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'childrens_survey.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children.csv"))) with open(os.path.join(temp_dir, "children.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons.csv"))) with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons_characters.csv"))) with open(os.path.join( temp_dir, "children_cartoons_characters.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons_characters.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir)
def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.csv".format(survey.name)))) with open(os.path.join(temp_dir, "{0}.csv".format(survey.name))) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'childrens_survey.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue(os.path.exists(os.path.join(temp_dir, "children.csv"))) with open(os.path.join(temp_dir, "children.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path('csvs', 'children.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists(os.path.join(temp_dir, "children_cartoons.csv"))) with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons_characters.csv"))) with open(os.path.join( temp_dir, "children_cartoons_characters.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open( _logger_fixture_path( 'csvs', 'children_cartoons_characters.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir)
def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets expected_sheet_names = [ 'childrens_survey', 'children', 'children_cartoons', 'children_cartoons_characters' ] self.assertEqual(wb.get_sheet_names(), expected_sheet_names) # check header columns main_sheet = wb.get_sheet_by_name('childrens_survey') expected_column_headers = [ u'name', u'age', u'geo/geolocation', u'geo/_geolocation_latitude', u'geo/_geolocation_longitude', u'geo/_geolocation_altitude', u'geo/_geolocation_precision', u'tel/tel.office', u'tel/tel.mobile', u'_id', u'meta/instanceID', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version' ] column_headers = [c[0].value for c in main_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) childrens_sheet = wb.get_sheet_by_name('children') expected_column_headers = [ u'children/name', u'children/age', u'children/fav_colors', u'children/fav_colors/red', u'children/fav_colors/blue', u'children/fav_colors/pink', u'children/ice.creams', u'children/ice.creams/vanilla', u'children/ice.creams/strawberry', u'children/ice.creams/chocolate', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version' ] column_headers = [c[0].value for c in childrens_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) cartoons_sheet = wb.get_sheet_by_name('children_cartoons') expected_column_headers = [ u'children/cartoons/name', u'children/cartoons/why', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version' ] column_headers = [c[0].value for c in cartoons_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) characters_sheet = wb.get_sheet_by_name('children_cartoons_characters') expected_column_headers = [ u'children/cartoons/characters/name', u'children/cartoons/characters/good_or_evil', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes', '_version' ] column_headers = [c[0].value for c in characters_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close()