def test_should_parse_csv_string_with_partial_values(self):
csv_data = """FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1, CL001, 11, Dr. A1,201
CLF1, CL001,, Dr. A1,201
CLF1,CL002,12,"Dr. LastName,Firstname" """
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(3, len(results))
form_code, values = results[0]
self.assertEqual("clf1", form_code)
self.assertEqual(
{"id": "CL001", "beds": "11",
"director": "Dr. A1",
"meds": "201"
}, values)
form_code, values = results[1]
self.assertEqual("clf1", form_code)
self.assertEqual(
{"id": "CL001", "beds": "",
"director": "Dr. A1",
"meds": "201"
}, values)
form_code, values = results[2]
self.assertEqual("clf1", form_code)
self.assertEqual(
{"id": "CL002", "beds": "12",
"director": "Dr. LastName,Firstname",
"meds": None
}, values)
def test_should_raise_exception_for_invalid_format(self):
empty_csv_data = """
"""
csv_parser = CsvParser()
with self.assertRaises(CSVParserInvalidHeaderFormatException):
results = csv_parser.parse(empty_csv_data)
def test_should_raise_exception_for_invalid_format(self):
empty_csv_data = """
"""
csv_parser = CsvParser()
with self.assertRaises(CSVParserInvalidHeaderFormatException):
results = csv_parser.parse(empty_csv_data)
def test_should_raise_exception_if_empty_field_in_header(self):
csv_data = """
FORM_CODE,ID, ,DIRECTOR,MEDS
CLF1,CL001,11,Dr. A1,201
CLF1,CL002,12,Dr. A2,202
CLF1,CL003,13,Dr. A3,203
CLF1,CL004,14,Dr. A4,204
CLF1,CL005,15,Dr. A5,205
"""
csv_parser = CsvParser()
with self.assertRaises(CSVParserInvalidHeaderFormatException):
csv_parser.parse(csv_data)
def test_should_raise_exception_if_empty_field_in_header(self):
csv_data = """
FORM_CODE,ID, ,DIRECTOR,MEDS
CLF1,CL001,11,Dr. A1,201
CLF1,CL002,12,Dr. A2,202
CLF1,CL003,13,Dr. A3,203
CLF1,CL004,14,Dr. A4,204
CLF1,CL005,15,Dr. A5,205
"""
csv_parser = CsvParser()
with self.assertRaises(CSVParserInvalidHeaderFormatException):
csv_parser.parse(csv_data)
def test_should_import_csv_string_with_extra_field_values(self):
csv_data = """FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1,CL001,11,Dr. A1,201
CLF1,CL002,12,Dr. A2,202,extra field
CLF1,CL003,13,Dr. A3,203,
CLF1,CL004,14,Dr. A4,204,extra field1, extra field 2
CLF1,CL005,15,Dr. A5,205,
"""
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(5, len(results))
self._assert_results(results)
def test_should_import_csv_string_with_extra_field_values(self):
csv_data = """FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1,CL001,11,Dr. A1,201
CLF1,CL002,12,Dr. A2,202,extra field
CLF1,CL003,13,Dr. A3,203,
CLF1,CL004,14,Dr. A4,204,extra field1, extra field 2
CLF1,CL005,15,Dr. A5,205,
"""
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(5, len(results))
self._assert_results(results)
def build(cls,
manager,
extension,
default_parser=None,
form_code=None,
is_datasender=False):
channels = dict({
".xls": Channel.XLS,
".xlsx": Channel.XLSX,
".csv": Channel.CSV
})
try:
channel = channels[extension]
except KeyError:
raise InvalidFileFormatException()
if default_parser is not None:
parser = default_parser()
elif extension == '.csv':
parser = CsvParser()
elif extension == '.xls':
parser = XlsParser()
elif extension == '.xlsx':
parser = XlsxParser()
else:
raise InvalidFileFormatException()
location_bridge = LocationBridge(
get_location_tree(), get_loc_hierarchy=get_location_hierarchy)
player = FilePlayer(manager,
parser,
channel,
location_tree=location_bridge,
is_datasender=is_datasender)
player.form_code = form_code
return player
def test_should_parse_csv_string_with_partial_values(self):
csv_data = """FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1, CL001, 11, Dr. A1,201
CLF1, CL001,, Dr. A1,201
CLF1,CL002,12,"Dr. LastName,Firstname" """
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(3, len(results))
form_code, values = results[0]
self.assertEqual("clf1", form_code)
self.assertEqual(
{
"id": "CL001",
"beds": "11",
"director": "Dr. A1",
"meds": "201"
}, values)
form_code, values = results[1]
self.assertEqual("clf1", form_code)
self.assertEqual(
{
"id": "CL001",
"beds": "",
"director": "Dr. A1",
"meds": "201"
}, values)
form_code, values = results[2]
self.assertEqual("clf1", form_code)
self.assertEqual(
{
"id": "CL002",
"beds": "12",
"director": "Dr. LastName,Firstname",
"meds": None
}, values)
def test_should_import_csv_string(self):
self._build_fixtures()
with patch("datawinners.utils.get_organization_from_manager"
) as get_organization:
organization = Mock(spec=Organization)
organization.in_trial_mode = True
get_organization.return_value = organization
player = FilePlayer(self.manager, CsvParser(), Channel.CSV)
response = player.accept(UPLOAD_DATA)
self.assertEqual([True, True, False, True, True],
[item.success for item in response])
def test_should_parse_csv_string_with_empty_lines(self):
csv_data = """
FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1, CL001, 11, Dr. A1,201
CLF1,CL002,12,Dr. A2,202
CLF1,CL003,13,Dr. A3,203
CLF1,CL004,14,Dr. A4,204
CLF1,CL005,15,Dr. A5,205
"""
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(5, len(results))
self._assert_results(results)
def test_should_parse_csv_string(self):
csv_data = """FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1, CL001, 11, Dr. A1,201
CLF1,CL002,12,Dr. A2,202
CLF1,CL003,13,Dr. A3,203
CLF1,CL004,14,Dr. A4,204
CLF1,CL005,15,Dr. A5,205"""
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(5, len(results))
for index, result in enumerate(results):
form_code, values = results[index]
self.assertEqual("clf1", form_code)
offset = index + 1
self.assertEqual(
{"id": "CL%03d" % (offset,), "beds": "%d" % (10 + offset,),
"director": "Dr. A%d" % (offset,),
"meds": "%d" % (200 + offset,)
}, values)
def test_should_parse_csv_string_with_empty_lines(self):
csv_data = """
FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1, CL001, 11, Dr. A1,201
CLF1,CL002,12,Dr. A2,202
CLF1,CL003,13,Dr. A3,203
CLF1,CL004,14,Dr. A4,204
CLF1,CL005,15,Dr. A5,205
"""
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(5, len(results))
self._assert_results(results)
def test_should_parse_csv_string(self):
csv_data = """FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1, CL001, 11, Dr. A1,201
CLF1,CL002,12,Dr. A2,202
CLF1,CL003,13,Dr. A3,203
CLF1,CL004,14,Dr. A4,204
CLF1,CL005,15,Dr. A5,205"""
csv_parser = CsvParser()
results = csv_parser.parse(csv_data)
self.assertEqual(5, len(results))
for index, result in enumerate(results):
form_code, values = results[index]
self.assertEqual("clf1", form_code)
offset = index + 1
self.assertEqual(
{
"id": "CL%03d" % (offset, ),
"beds": "%d" % (10 + offset, ),
"director": "Dr. A%d" % (offset, ),
"meds": "%d" % (200 + offset, )
}, values)
def _handle_uploaded_file(file_name, file, manager):
base_name, extension = os.path.splitext(file_name)
if extension == '.csv':
csv_player = CsvPlayer(dbm=manager,
submission_handler=SubmissionHandler(manager),
parser=CsvParser())
response = csv_player.accept(file)
elif extension == '.xls':
xls_player = XlsPlayer(dbm=manager,
submission_handler=SubmissionHandler(manager),
parser=XlsParser())
response = xls_player.accept(file)
else:
raise InvalidFileFormatException()
return response
def setUp(self):
self.dbm = Mock(spec=DatabaseManager)
loc_tree = Mock()
loc_tree.get_hierarchy_path.return_value = None
self.submission_handler_mock = Mock(spec=SubmissionHandler)
self.parser = CsvParser()
self.csv_data = """
FORM_CODE,ID,BEDS,DIRECTOR,MEDS
CLF1,CL001,10,Dr. A,201
CLF1,CL002,11,Dr. B,202
CLF2,CL003,12,Dr. C,203
CLF1,CL004,13,Dr. D,204
CLF1,CL005,14,Dr. E,205
"""
self._mock_short_code_generator()
self.player = CsvPlayer(self.dbm, self.submission_handler_mock,
self.parser, loc_tree)
def setUp(self):
MangroveTestCase.setUp(self)
initializer.run(self.manager)
self.entity_type = ["reporter"]
self.telephone_number_type = DataDictType(self.manager, name='telephone_number', slug='telephone_number',
primitive_type='string')
self.entity_id_type = DataDictType(self.manager, name='Entity Id Type', slug='entity_id', primitive_type='string')
self.name_type = DataDictType(self.manager, name='Name', slug='name', primitive_type='string')
self.telephone_number_type.save()
self.name_type.save()
self.reporter = create_entity(self.manager, entity_type=self.entity_type,
location=["India", "Pune"], aggregation_paths=None, short_code="rep1",
)
self.reporter.add_data(data=[(MOBILE_NUMBER_FIELD, '1234', self.telephone_number_type),
(NAME_FIELD, "Test_reporter", self.name_type)], submission=dict(submission_id="1"))
question1 = TextField(name="entity_question", code="EID", label="What is associated entity",
entity_question_flag=True, ddtype=self.entity_id_type)
question2 = TextField(name="Name", code="NAME", label="Clinic Name",
defaultValue="some default value",
ddtype=self.name_type, required=False)
self.form_model = FormModel(self.manager, entity_type=self.entity_type, name="Dengue", label="Dengue form_model",
form_code="clinic", type='survey', fields=[question1,question2])
self.form_model.save()
self.csv_data_for_activity_report = """
FORM_CODE,EID,NAME
clinic,rep1,XYZ
"""
self.csv_data_about_reporter = """
FORM_CODE,t,n,l,d,m
REG,"reporter",Dr. A,Pune,"Description",201
"""
self.csv_data_with_same_mobile_number = """
FORM_CODE,t,n,l,d,m
REG,"reporter",Dr. A,Pune,"Description",201
REG,"reporter",Dr. B,Pune,"Description",201
"""
self.csv_data_with_exception = """
FORM_CODE,t,n,l,d,m
REG,"reporter",Dr. A,Pune,"Description",201
REG,"reporter",Dr. B,Pune,"Description",201
REG,"reporter",Dr. C,Pune,"Description",202
"""
self.csv_data_with_missing_name = """
FORM_CODE,t,n,l,d,m
REG,"reporter",,Pune,"Description",201
"""
self.csv_data_with_missing_type = """
FORM_CODE,t,n,l,d,m
REG,,Dr. A,Pune,"Description",201
"""
self.csv_data_with_incorrect_mobile_number = """
FORM_CODE,t,n,l,d,m
REG,"reporter",Dr. A,Pune,"Description",2014678447676512
REG,"reporter",Dr. A,Pune,"Description",~!@#$%^&*()+|}
REG,"reporter",Dr. A,Pune,"Description",
"""
self.csv_data_with_incorrect_GPS = """
FORM_CODE,t,n,g,d,m
REG,"reporter",Dr. A,18,"Description",201
"""
self.csv_data_with_out_of_range_GPS_value = """
FORM_CODE,t,n,g,d,m
REG,"reporter",Dr. A,-95 48,"Description",201
REG,"reporter",Dr. A,-18 184,"Description",201
"""
self.csv_data_without_form_code= """
FORM_CODE,t,n,g,d,m
,"reporter",Dr. A,-95 48,"Description",201
ABC,"reporter",Dr. A,-95 48,"Description",201
"""
self.parser = CsvParser()
self.file_player = FilePlayer(self.manager,self.parser, Channel.CSV, LocationBridge(DummyLocationTree(),dummy_get_location_hierarchy))