def test_real_csv_file_2(self) -> None:
csv_processor = CsvProcessor(etk=etk,
heading_row=2,
content_start_row=10,
content_end_row=13)
file_path = 'etk/unit_tests/ground_truth/masie_4km_allyears_extent_sqkm.csv'
test_docs = [
doc.cdr_document
for doc in csv_processor.tabular_extractor(filename=file_path,
dataset='test_set')
]
expected_docs = [{
'yyyyddd': 2006008,
' (0) Northern_Hemisphere': '13536736.84',
' (1) Beaufort_Sea': ' 1069710.81',
' (2) Chukchi_Sea': ' 966006.16',
' (3) East_Siberian_Sea': ' 1087102.72',
' (4) Laptev_Sea': ' 897773.37',
' (5) Kara_Sea': ' 927602.17',
' (6) Barents_Sea': ' 474574.82',
' (7) Greenland_Sea': ' 590029.18',
' (8) Baffin_Bay_Gulf_of_St._Lawrence': ' 1005790.38',
' (9) Canadian_Archipelago': ' 852715.31',
' (10) Hudson_Bay': ' 1260779.00',
' (11) Central_Arctic': ' 3240326.47',
' (12) Bering_Sea': ' 692832.54',
' (13) Baltic_Sea': ' 21327.46',
' (14) Sea_of_Okhotsk': ' 424563.54',
' (15) Yellow_Sea': ' 14830.45',
' (16) Cook_Inlet': ' 8202.95',
'file_name':
'etk/unit_tests/ground_truth/masie_4km_allyears_extent_sqkm.csv',
'dataset': 'test_set'
}, {
'yyyyddd': 2006009,
' (0) Northern_Hemisphere': '13536887.64',
' (1) Beaufort_Sea': ' 1069710.81',
' (2) Chukchi_Sea': ' 966006.16',
' (3) East_Siberian_Sea': ' 1087102.72',
' (4) Laptev_Sea': ' 897773.37',
' (5) Kara_Sea': ' 927602.17',
' (6) Barents_Sea': ' 474574.82',
' (7) Greenland_Sea': ' 590029.18',
' (8) Baffin_Bay_Gulf_of_St._Lawrence': ' 1005790.38',
' (9) Canadian_Archipelago': ' 852715.31',
' (10) Hudson_Bay': ' 1260779.00',
' (11) Central_Arctic': ' 3240326.47',
' (12) Bering_Sea': ' 692832.54',
' (13) Baltic_Sea': ' 21478.25',
' (14) Sea_of_Okhotsk': ' 424563.54',
' (15) Yellow_Sea': ' 14830.45',
' (16) Cook_Inlet': ' 8202.95',
'file_name':
'etk/unit_tests/ground_truth/masie_4km_allyears_extent_sqkm.csv',
'dataset': 'test_set'
}, {
'yyyyddd': 2006010,
' (0) Northern_Hemisphere': '13505426.35',
' (1) Beaufort_Sea': ' 1069710.81',
' (2) Chukchi_Sea': ' 966006.16',
' (3) East_Siberian_Sea': ' 1087102.72',
' (4) Laptev_Sea': ' 897773.37',
' (5) Kara_Sea': ' 933999.29',
' (6) Barents_Sea': ' 448185.27',
' (7) Greenland_Sea': ' 588279.64',
' (8) Baffin_Bay_Gulf_of_St._Lawrence': ' 1016857.87',
' (9) Canadian_Archipelago': ' 852715.31',
' (10) Hudson_Bay': ' 1260779.00',
' (11) Central_Arctic': ' 3217380.82',
' (12) Bering_Sea': ' 705348.17',
' (13) Baltic_Sea': ' 21493.81',
' (14) Sea_of_Okhotsk': ' 414191.19',
' (15) Yellow_Sea': ' 14830.45',
' (16) Cook_Inlet': ' 8202.95',
'file_name':
'etk/unit_tests/ground_truth/masie_4km_allyears_extent_sqkm.csv',
'dataset': 'test_set'
}, {
'yyyyddd': 2006011,
' (0) Northern_Hemisphere': '13493030.93',
' (1) Beaufort_Sea': ' 1069710.81',
' (2) Chukchi_Sea': ' 966006.16',
' (3) East_Siberian_Sea': ' 1087102.72',
' (4) Laptev_Sea': ' 897773.37',
' (5) Kara_Sea': ' 933999.29',
' (6) Barents_Sea': ' 448185.27',
' (7) Greenland_Sea': ' 588279.64',
' (8) Baffin_Bay_Gulf_of_St._Lawrence': ' 1013091.28',
' (9) Canadian_Archipelago': ' 852715.31',
' (10) Hudson_Bay': ' 1260779.00',
' (11) Central_Arctic': ' 3217380.82',
' (12) Bering_Sea': ' 696719.33',
' (13) Baltic_Sea': ' 21493.81',
' (14) Sea_of_Okhotsk': ' 414191.19',
' (15) Yellow_Sea': ' 14830.45',
' (16) Cook_Inlet': ' 8202.95',
'file_name':
'etk/unit_tests/ground_truth/masie_4km_allyears_extent_sqkm.csv',
'dataset': 'test_set'
}]
self.assertEqual(test_docs, expected_docs)
def test_real_excel_with_sheetname(self) -> None:
csv_processor = CsvProcessor(etk=etk,
heading_row=1,
content_start_row=10,
content_end_row=12)
file_path = 'etk/unit_tests/ground_truth/NST-Main Sheet.xlsx'
sheets_name = 'NST Main Dataset'
test_docs = [
doc.cdr_document for doc in csv_processor.tabular_extractor(
filename=file_path, sheet_name=sheets_name, dataset='test_set')
]
expected_docs = [{
'Title': 'David Usman and Shot Dead',
'Date': '2011-06-07',
'Community (city,town, ward)': 'Maiduguri',
'LGA': 'Maiduguri',
'State': 'Borno',
'Total Deaths': 2,
'Boko Haram (P)': 'Boko Haram',
'State Actor (P)': '',
'Sectarian Actor (excluding BH) (P)': '',
'Other Armed Actor (P)': '',
'Kidnapper (P)': '',
'Robber (P)': '',
'Other (P)': '',
'Election-related Actor (P)': '',
'Cameroon State Actor (P)': '',
'Boko Haram (V)': '',
'State Actor (V)': '',
'Sectarian Actor (V)': 2,
'Other Armed Actor (V)': '',
'Political Actor (V)': '',
'Kidnapper (V)': '',
'Kidnapee (V)': '',
'Robber (V)': '',
'Journalist (V)': '',
'Civilian (V)': '',
'Election-related Actor (V)': '',
'Cameroon State Actor': '',
'Bomb': '',
'Gun': 'Gun',
'Machete': '',
'Suicide Bombing': '',
'Other Weapon': '',
'TK': 'Targeted Killing',
'Drinking Establishment': '',
'Goverment Building': '',
'Church': '',
'Mosque': '',
'Bank': '',
'School': '',
'Other Location': 'Other',
'Notes': '',
'Sources 1': 'http://allafrica.com/stories/201106100373.html',
'Sources 2': 'http://www.bbc.co.uk/news/world-africa-13724349',
'Sources 3': '',
'Latitude': '',
'Longitude': '',
'full place name': 'Maiduguri, Borno, Nigeria',
'country': 'Nigeria',
'file_name': 'etk/unit_tests/ground_truth/NST-Main Sheet.xlsx',
'dataset': 'test_set'
}, {
'Title': 'Explosion, Firefight at Gwange Police Station',
'Date': '2011-06-07',
'Community (city,town, ward)': 'Gwange Police Station, Maiduguri',
'LGA': 'Maiduguri',
'State': 'Borno',
'Total Deaths': 3,
'Boko Haram (P)': 'Boko Haram',
'State Actor (P)': 'State Actor',
'Sectarian Actor (excluding BH) (P)': '',
'Other Armed Actor (P)': '',
'Kidnapper (P)': '',
'Robber (P)': '',
'Other (P)': '',
'Election-related Actor (P)': '',
'Cameroon State Actor (P)': '',
'Boko Haram (V)': 3,
'State Actor (V)': '',
'Sectarian Actor (V)': '',
'Other Armed Actor (V)': '',
'Political Actor (V)': '',
'Kidnapper (V)': '',
'Kidnapee (V)': '',
'Robber (V)': '',
'Journalist (V)': '',
'Civilian (V)': '',
'Election-related Actor (V)': '',
'Cameroon State Actor': '',
'Bomb': 'Bomb',
'Gun': 'Gun',
'Machete': '',
'Suicide Bombing': '',
'Other Weapon': '',
'TK': '',
'Drinking Establishment': '',
'Goverment Building': 'Government Building',
'Church': '',
'Mosque': '',
'Bank': '',
'School': '',
'Other Location': '',
'Notes': '',
'Sources 1':
'http://www.google.com/hostednews/afp/article/ALeqM5hofvKayKKAFFtiX9-Ic5bG2ptVmg?docId=CNG.fafcacea0287fbeab90256732f165e1e.771',
'Sources 2':
'http://news.xinhuanet.com/english2010/world/2011-06/08/c_13915959.htm',
'Sources 3': '',
'Latitude': '',
'Longitude': '',
'full place name': 'Maiduguri, Borno, Nigeria',
'country': 'Nigeria',
'file_name': 'etk/unit_tests/ground_truth/NST-Main Sheet.xlsx',
'dataset': 'test_set'
}, {
'Title': 'Explosions at Dandal Police Station',
'Date': '2011-06-07',
'Community (city,town, ward)': 'Dandal Police Station, Maiduguri',
'LGA': 'Maiduguri',
'State': 'Borno',
'Total Deaths': 0,
'Boko Haram (P)': 'Boko Haram',
'State Actor (P)': '',
'Sectarian Actor (excluding BH) (P)': '',
'Other Armed Actor (P)': '',
'Kidnapper (P)': '',
'Robber (P)': '',
'Other (P)': '',
'Election-related Actor (P)': '',
'Cameroon State Actor (P)': '',
'Boko Haram (V)': '',
'State Actor (V)': 0,
'Sectarian Actor (V)': '',
'Other Armed Actor (V)': '',
'Political Actor (V)': '',
'Kidnapper (V)': '',
'Kidnapee (V)': '',
'Robber (V)': '',
'Journalist (V)': '',
'Civilian (V)': '',
'Election-related Actor (V)': '',
'Cameroon State Actor': '',
'Bomb': 'Bomb',
'Gun': '',
'Machete': '',
'Suicide Bombing': '',
'Other Weapon': '',
'TK': '',
'Drinking Establishment': '',
'Goverment Building': 'Government Building',
'Church': '',
'Mosque': '',
'Bank': '',
'School': '',
'Other Location': '',
'Notes': '',
'Sources 1':
'http://news.xinhuanet.com/english2010/world/2011-06/08/c_13915959.htm',
'Sources 2':
'http://www.google.com/hostednews/afp/article/ALeqM5hofvKayKKAFFtiX9-Ic5bG2ptVmg?docId=CNG.fafcacea0287fbeab90256732f165e1e.771',
'Sources 3': '',
'Latitude': '',
'Longitude': '',
'full place name': 'Maiduguri, Borno, Nigeria',
'country': 'Nigeria',
'file_name': 'etk/unit_tests/ground_truth/NST-Main Sheet.xlsx',
'dataset': 'test_set'
}]
self.assertEqual(test_docs, expected_docs)
def test_real_csv_file_1(self) -> None:
csv_processor = CsvProcessor(etk=etk,
heading_row=1,
content_end_row=4,
ends_with_blank_row=False)
file_path = 'etk/unit_tests/ground_truth/acled_raw_data.csv'
test_docs = [
doc.cdr_document
for doc in csv_processor.tabular_extractor(filename=file_path,
dataset='test_set')
]
expected_docs = [{
'data_id':
336907,
'iso':
180,
'event_id_cnty':
'DRC11776',
'event_id_no_cnty':
11776,
'event_date':
'2018-01-13',
'year':
2018,
'time_precision':
1,
'event_type':
'Battle-No change of territory',
'actor1':
'Military Forces of Democratic Republic of Congo (2001-)',
'assoc_actor_1':
'',
'inter1':
1,
'actor2':
'ADF: Allied Democratic Forces',
'assoc_actor_2':
'',
'inter2':
2,
'interaction':
12,
'region':
'Central Africa',
'country':
'Democratic Republic of Congo',
'admin1':
'Nord-Kivu',
'admin2':
'Nord-Kivu',
'admin3':
'Oicha',
'location':
'Oicha',
'latitude':
'0.7',
'longitude':
'29.5167',
'geo_precision':
1,
'source':
'Radio Okapi',
'source_scale':
'Subnational',
'notes':
"Presumed FARDC attacked the ADF in the periphery of Oicha on January 13th. Shots were heard "
"in the locality and it is suspected that the FARDC are attacking the 'death triangle' situated in between "
"Mbau, Kamango and Eringeti. The reports are not confirmed by the military.",
'fatalities':
0,
'timestamp':
1516117305,
'file_name':
'etk/unit_tests/ground_truth/acled_raw_data.csv',
'dataset':
'test_set'
}, {
'data_id':
336908,
'iso':
180,
'event_id_cnty':
'DRC11777',
'event_id_no_cnty':
11777,
'event_date':
'2018-01-13',
'year':
2018,
'time_precision':
1,
'event_type':
'Battle-No change of territory',
'actor1':
'Military Forces of Democratic Republic of Congo (2001-)',
'assoc_actor_1':
'',
'inter1':
1,
'actor2':
'ADF: Allied Democratic Forces',
'assoc_actor_2':
'',
'inter2':
2,
'interaction':
12,
'region':
'Central Africa',
'country':
'Democratic Republic of Congo',
'admin1':
'Nord-Kivu',
'admin2':
'Beni',
'admin3':
'Beni',
'location':
'Beni',
'latitude':
'0.49658',
'longitude':
'29.4654',
'geo_precision':
1,
'source':
'Reuters; Radio Okapi',
'source_scale':
'Subnational',
'notes':
'The FARDC launched, on January 13th, an offensive against the ADF in Beni and Lubero, '
'in response to the recents attacks by the group. Gunfires and explosions were heard '
'in Beni all throughout Saturday (13th).',
'fatalities':
0,
'timestamp':
1516117305,
'file_name':
'etk/unit_tests/ground_truth/acled_raw_data.csv',
'dataset':
'test_set'
}, {
'data_id':
336909,
'iso':
180,
'event_id_cnty':
'DRC11778',
'event_id_no_cnty':
11778,
'event_date':
'2018-01-13',
'year':
2018,
'time_precision':
1,
'event_type':
'Battle-No change '
'of territory',
'actor1':
'Military Forces of Democratic Republic of Congo (2001-)',
'assoc_actor_1':
'',
'inter1':
1,
'actor2':
'ADF: Allied Democratic Forces',
'assoc_actor_2':
'',
'inter2':
2,
'interaction':
12,
'region':
'Central Africa',
'country':
'Democratic Republic of Congo',
'admin1':
'Nord-Kivu',
'admin2':
'Nord-Kivu',
'admin3':
'Lubero',
'location':
'Lubero',
'latitude':
'-0.15867',
'longitude':
'29.2386',
'geo_precision':
1,
'source':
'Reuters; Radio Okapi',
'source_scale':
'Subnational',
'notes':
'The FARDC launched, on January 13th, an offensive '
'against the ADF in Beni and Lubero, in response to the recents attacks by the group.',
'fatalities':
0,
'timestamp':
1516117305,
'file_name':
'etk/unit_tests/ground_truth/acled_raw_data.csv',
'dataset':
'test_set'
}]
self.assertEqual(test_docs, expected_docs)