def test_nested_geo_paths_csv(self):
self.xls_file_path = os.path.join(self.fixtures_dir, "tutorial-nested-geo.xls")
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
good_csv = open(os.path.join(self.fixtures_dir, "another_good.csv"))
csv_import.submit_csv(self.user.username, self.xform, good_csv)
self.assertEqual(Instance.objects.count(), 9, u"submit_csv edits #1 test Failed!")
def test_csv_with_multiple_select_in_one_column(self):
self.xls_file_path = os.path.join(self.fixtures_dir, "form_with_multiple_select.xlsx")
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
good_csv = open(os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv"))
csv_import.submit_csv(self.user.username, self.xform, good_csv)
self.assertEqual(Instance.objects.count(), 1, u"submit_csv edits #1 test Failed!")
def test_submit_csv_and_rollback(self):
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(),
count + 9, u'submit_csv test Failed!')
# Check that correct # of submissions belong to our user
self.assertEqual(Instance.objects.filter(user=self.user).count(),
count + 8, u'submit_csv username check failed!')
def test_excel_date_conversion(self):
"""Convert date from 01/01/1900 to 01-01-1900"""
date_md_form = """
| survey |
| | type | name | label |
| | today | today | Today |
| | text | name | Name |
| | date | tdate | Date |
| | start | start | |
| | end | end | |
| | dateTime | now | Now |
| choices |
| | list name | name | label |
| settings |
| | form_title | form_id |
| | Dates | dates |
"""
self._create_user_and_login()
data = {'name': 'data'}
survey = self.md_to_pyxform_survey(date_md_form, kwargs=data)
survey['sms_keyword'] = survey['id_string']
project = get_user_default_project(self.user)
xform = XForm(created_by=self.user, user=self.user,
xml=survey.to_xml(), json=survey.to_json(),
project=project)
xform.save()
date_csv = open(os.path.join(
self.fixtures_dir, 'date.csv'), 'rb')
date_csv.seek(0)
csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig')
xl_dates = []
xl_datetime = []
# xl dates
for row in csv_reader:
xl_dates.append(row.get('tdate'))
xl_datetime.append(row.get('now'))
csv_import.submit_csv(self.user.username, xform, date_csv)
# converted dates
conv_dates = [instance.json.get('tdate')
for instance in Instance.objects.filter(
xform=xform).order_by('date_created')]
conv_datetime = [instance.json.get('now')
for instance in Instance.objects.filter(
xform=xform).order_by('date_created')]
self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019'])
self.assertEqual(
xl_datetime,
[u'6/12/2020 13:20', u'2019-03-11T16:00:51.147+02:00'])
self.assertEqual(
conv_datetime,
[u'2020-06-12T13:20:00.000000', u'2019-03-11T16:00:51.147+02:00'])
self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26'])
def test_data_upload(self):
"""Data upload for submissions with no uuids"""
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
count = Instance.objects.count()
single_csv = open(os.path.join(
self.fixtures_dir, 'single_data_upload.csv'), 'rb')
csv_import.submit_csv(self.user.username, self.xform, single_csv)
self.xform.refresh_from_db()
self.assertEqual(self.xform.num_of_submissions, count + 1)
def test_nested_geo_paths_csv(self):
self.xls_file_path = os.path.join(self.fixtures_dir,
'tutorial-nested-geo.xls')
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv'),
'rb')
csv_import.submit_csv(self.user.username, self.xform, good_csv)
self.assertEqual(Instance.objects.count(), 9,
'submit_csv edits #1 test Failed!')
def test_submit_csv_and_rollback(self):
xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.get()
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(), count + 9, u"submit_csv test Failed!")
# Check that correct # of submissions belong to our user
self.assertEqual(
Instance.objects.filter(user=self.user).count(), count + 8, u"submit_csv username check failed!"
)
def test_csv_with_multiple_select_in_one_column(self):
self.xls_file_path = os.path.join(self.fixtures_dir,
'form_with_multiple_select.xlsx')
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
good_csv = open(
os.path.join(self.fixtures_dir,
'csv_import_with_multiple_select.csv'))
csv_import.submit_csv(self.user.username, self.xform, good_csv)
self.assertEqual(Instance.objects.count(), 1,
u'submit_csv edits #1 test Failed!')
def test_data_upload(self, send_message_mock):
"""Data upload for submissions with no uuids"""
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
count = Instance.objects.count()
single_csv = open(os.path.join(
self.fixtures_dir, 'single_data_upload.csv'), 'rb')
csv_import.submit_csv(self.user.username, self.xform, single_csv)
self.xform.refresh_from_db()
self.assertEqual(self.xform.num_of_submissions, count + 1)
# message sent upon submission creation
self.assertTrue(send_message_mock.called)
send_message_mock.called_with(self.xform.id, XFORM, SUBMISSION_CREATED)
def test_csv_with_repeats_import(self):
self.xls_file_path = os.path.join(self.this_directory, 'fixtures',
'csv_export',
'tutorial_w_repeats.xls')
repeats_csv = open(
os.path.join(self.this_directory, 'fixtures', 'csv_export',
'tutorial_w_repeats.csv'))
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
pre_count = self.xform.instances.count()
csv_import.submit_csv(self.user.username, self.xform, repeats_csv)
count = self.xform.instances.count()
self.assertEqual(count, 1 + pre_count)
def test_csv_with_repeats_import(self):
self.xls_file_path = os.path.join(self.this_directory, 'fixtures',
'csv_export',
'tutorial_w_repeats.xls')
repeats_csv = open(
os.path.join(self.this_directory, 'fixtures', 'csv_export',
'tutorial_w_repeats.csv'),
'rb')
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
pre_count = self.xform.instances.count()
csv_import.submit_csv(self.user.username, self.xform, repeats_csv)
count = self.xform.instances.count()
self.assertEqual(count, 1 + pre_count)
def test_submit_csv_and_rollback(self):
xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main",
"tests", "fixtures", "tutorial.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.get()
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(), count + 9,
u'submit_csv test Failed!')
# Check that correct # of submissions belong to our user
self.assertEqual(
Instance.objects.filter(user=self.user).count(), count + 8,
u'submit_csv username check failed!')
def test_submit_csv_edits(self):
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(),
9, 'submit_csv edits #1 test Failed!')
edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv'))
edit_csv_str = edit_csv.read()
edit_csv = StringIO(edit_csv_str.format(
*[x.get('uuid') for x in Instance.objects.values('uuid')]))
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, edit_csv)
self.assertEqual(Instance.objects.count(),
count, 'submit_csv edits #2 test Failed!')
def test_submit_csv_edits(self):
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(),
9, u'submit_csv edits #1 test Failed!')
edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv'))
edit_csv_str = edit_csv.read()
edit_csv = StringIO(edit_csv_str.format(
*[x.get('uuid') for x in Instance.objects.values('uuid')]))
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, edit_csv)
self.assertEqual(Instance.objects.count(), count,
u'submit_csv edits #2 test Failed!')
def test_submit_csv_xml_location_property_test(self, d2x, safe_create_instance):
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
safe_create_instance.return_value = [None]
single_csv = open(os.path.join(self.fixtures_dir, "single.csv"))
csv_import.submit_csv(self.user.username, self.xform, single_csv)
test_location_val = "83.3595 -32.8601 0 1"
test_location2_val = "21.22474 -10.5601 50000 200"
self.assertNotEqual(d2x.call_args, None, u"dict2xmlsubmission not called")
call_dict = d2x.call_args[0][0]
self.assertEqual(call_dict.get("test_location"), test_location_val, u"Location prop test fail")
self.assertEqual(call_dict.get("test_location2"), test_location2_val, u"Location2 prop test fail")
def test_submit_csv_edits(self):
xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.get()
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(), 9, u"submit_csv edits #1 test Failed!")
edit_csv = open(os.path.join(self.fixtures_dir, "edit.csv"))
edit_csv_str = edit_csv.read()
edit_csv = StringIO(edit_csv_str.format(*[x.get("uuid") for x in Instance.objects.values("uuid")]))
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, edit_csv)
self.assertEqual(Instance.objects.count(), count, u"submit_csv edits #2 test Failed!")
def test_reject_spaces_in_headers(self):
non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv'))
result = csv_import.submit_csv(self.user.username,
self.xform, non_utf8csv)
self.assertEqual(result.get('error'),
u'CSV file fieldnames should not contain spaces',
u'Incorrect error message returned.')
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
resp = {}
if request.method == 'GET':
resp.update(get_async_csv_submission_status(
request.QUERY_PARAMS.get('job_uuid')))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
else:
num_rows = sum(1 for row in csv_file) - 1
if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD:
resp.update(submit_csv(request.user.username,
self.get_object(), csv_file))
else:
resp.update(
{u'task_id': submit_csv_async.delay(
request.user.username, self.get_object(),
csv_file).task_id})
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
def test_reject_spaces_in_headers(self):
non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv'))
result = csv_import.submit_csv(self.user.username, self.xform,
non_utf8csv)
self.assertEqual(result.get('error'),
u'CSV file fieldnames should not contain spaces',
u'Incorrect error message returned.')
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
resp = {}
if request.method == 'GET':
resp.update(
get_async_csv_submission_status(
request.QUERY_PARAMS.get('job_uuid')))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
else:
num_rows = sum(1 for row in csv_file) - 1
if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD:
resp.update(
submit_csv(request.user.username, self.get_object(),
csv_file))
else:
resp.update({
u'task_id':
submit_csv_async.delay(request.user.username,
self.get_object(),
csv_file).task_id
})
return Response(data=resp,
status=status.HTTP_200_OK if resp.get('error') is None
else status.HTTP_400_BAD_REQUEST)
def test_enforces_data_type_and_rollback(self):
"""
Test that data type constraints are enforced and instances created
during the process are rolled back
"""
# Test integer constraint is enforced
xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main",
"tests", "fixtures", "tutorial.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.last()
bad_data = open(
os.path.join(self.fixtures_dir, 'bad_data.csv'),
'rb')
count = Instance.objects.count()
result = csv_import.submit_csv(self.user.username, self.xform,
bad_data)
expected_error = (
"Invalid CSV data imported in row(s): {1: ['Unknown date format(s)"
": 2014-0903', 'Unknown datetime format(s): sdsa', "
"'Unknown integer format(s): 21.53'], 2: ['Unknown integer format"
"(s): 22.32'], 4: ['Unknown date format(s): 2014-0900'], "
"5: ['Unknown date format(s): 2014-0901'], 6: ['Unknown "
"date format(s): 2014-0902'], 7: ['Unknown date format(s):"
" 2014-0903']}")
self.assertEqual(
result.get('error'),
expected_error)
# Assert all created instances were rolled back
self.assertEqual(count, Instance.objects.count())
def test_submit_csv_xml_location_property_test(
self, d2x, safe_create_instance):
safe_create_instance.return_value = [None, ]
single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'))
csv_import.submit_csv(self.user.username, self.xform, single_csv)
test_location_val = '83.3595 -32.8601 0 1'
test_location2_val = '21.22474 -10.5601 50000 200'
self.assertNotEqual(d2x.call_args, None,
u'dict2xmlsubmission not called')
call_dict = d2x.call_args[0][0]
self.assertEqual(call_dict.get('test_location'),
test_location_val, u'Location prop test fail')
self.assertEqual(call_dict.get('test_location2'),
test_location2_val, u'Location2 prop test fail')
def test_reject_spaces_in_headers(self):
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
non_utf8csv = open(os.path.join(self.fixtures_dir, "header_space.csv"))
result = csv_import.submit_csv(self.user.username, self.xform, non_utf8csv)
self.assertEqual(
result.get("error"), u"CSV file fieldnames should not contain spaces", u"Incorrect error message returned."
)
def test_csv_with__more_than_4_repeats_import(self):
self.xls_file_path = os.path.join(self.this_directory, 'fixtures',
'csv_export',
'tutorial_w_repeats.xls')
repeats_csv = open(
os.path.join(self.this_directory, 'fixtures', 'csv_export',
'tutorial_w_repeats_import.csv'))
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
pre_count = self.xform.instances.count()
csv_import.submit_csv(self.user.username, self.xform, repeats_csv)
count = self.xform.instances.count()
self.assertEqual(count, 1 + pre_count)
instance = self.xform.instances.last()
# repeats should be 6
self.assertEqual(6, len(instance.json.get('children')))
def test_submit_csv_xml_params(self, safe_create_instance):
safe_create_instance.return_value = {}
single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'))
csv_import.submit_csv(self.user.username, self.xform, single_csv)
xml_file_param = StringIO(
open(os.path.join(self.fixtures_dir, 'single.xml')).read())
safe_create_args = list(safe_create_instance.call_args[0])
self.assertEqual(safe_create_args[0], self.user.username,
u'Wrong username passed')
self.assertEqual(strip_xml_uuid(safe_create_args[1].getvalue()),
strip_xml_uuid(xml_file_param.getvalue()),
u'Wrong xml param passed')
self.assertEqual(safe_create_args[2], [],
u'Wrong media array param passed')
self.assertEqual(safe_create_args[3], self.xform.uuid,
u'Wrong xform uuid passed')
self.assertEqual(safe_create_args[4], None)
def test_excel_date_conversion(self):
"""Convert date from 01/01/1900 to 01-01-1900"""
date_md_form = """
| survey |
| | type | name | label |
| | today | today | Today |
| | text | name | Name |
| | date | tdate | Date |
| choices |
| | list name | name | label |
| settings |
| | form_title | form_id |
| | Dates | dates |
"""
self._create_user_and_login()
data = {'name': 'data'}
survey = self.md_to_pyxform_survey(date_md_form, kwargs=data)
survey['sms_keyword'] = survey['id_string']
project = get_user_default_project(self.user)
xform = XForm(created_by=self.user,
user=self.user,
xml=survey.to_xml(),
json=survey.to_json(),
project=project)
xform.save()
date_csv = open(os.path.join(self.fixtures_dir, 'date.csv'), 'rb')
date_csv.seek(0)
csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig')
xl_dates = []
# xl dates
for row in csv_reader:
xl_dates.append(row.get('tdate'))
csv_import.submit_csv(self.user.username, xform, date_csv)
# converted dates
conv_dates = [
instance.json.get('tdate') for instance in Instance.objects.filter(
xform=xform).order_by('date_created')
]
self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019'])
self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26'])
def test_import_non_utf8_csv(self):
count = Instance.objects.count()
non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv'))
result = csv_import.submit_csv(self.user.username, self.xform,
non_utf8_csv)
self.assertEqual(result.get('error'),
u'CSV file must be utf-8 encoded',
u'Incorrect error message returned.')
self.assertEqual(Instance.objects.count(), count,
u'Non unicode csv import rollback failed!')
def test_csv_with__more_than_4_repeats_import(self):
self.xls_file_path = os.path.join(self.this_directory, 'fixtures',
'csv_export',
'tutorial_w_repeats.xls')
repeats_csv = open(
os.path.join(self.this_directory, 'fixtures', 'csv_export',
'tutorial_w_repeats_import.csv'),
'rb')
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
pre_count = self.xform.instances.count()
csv_import.submit_csv(self.user.username, self.xform, repeats_csv)
count = self.xform.instances.count()
self.assertEqual(count, 1 + pre_count)
instance = self.xform.instances.last()
# repeats should be 6
self.assertEqual(6, len(instance.json.get('children')))
def test_submit_csv_xml_params(self, safe_create_instance):
safe_create_instance.return_value = {}
single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'))
csv_import.submit_csv(self.user.username, self.xform, single_csv)
xml_file_param = StringIO(open(os.path.join(self.fixtures_dir,
'single.xml')).read())
safe_create_args = list(safe_create_instance.call_args[0])
self.assertEqual(safe_create_args[0],
self.user.username,
u'Wrong username passed')
self.assertEqual(strip_xml_uuid(safe_create_args[1].getvalue()),
strip_xml_uuid(xml_file_param.getvalue()),
u'Wrong xml param passed')
self.assertEqual(safe_create_args[2], [],
u'Wrong media array param passed')
self.assertEqual(safe_create_args[3], self.xform.uuid,
u'Wrong xform uuid passed')
self.assertEqual(safe_create_args[4], None)
def test_import_non_utf8_csv(self):
xls_file_path = os.path.join(self.fixtures_dir, "mali_health.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.get()
count = Instance.objects.count()
non_utf8_csv = open(os.path.join(self.fixtures_dir, "non_utf8.csv"))
result = csv_import.submit_csv(self.user.username, self.xform, non_utf8_csv)
self.assertEqual(result.get("error"), u"CSV file must be utf-8 encoded", u"Incorrect error message returned.")
self.assertEqual(Instance.objects.count(), count, u"Non unicode csv import rollback failed!")
def test_import_non_utf8_csv(self):
count = Instance.objects.count()
non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv'))
result = csv_import.submit_csv(self.user.username,
self.xform, non_utf8_csv)
self.assertEqual(result.get('error'),
u'CSV file must be utf-8 encoded',
u'Incorrect error message returned.')
self.assertEqual(Instance.objects.count(), count,
u'Non unicode csv import rollback failed!')
def test_submit_csv_xml_location_property_test(self, d2x,
safe_create_instance):
safe_create_instance.return_value = [
None,
]
single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'))
csv_import.submit_csv(self.user.username, self.xform, single_csv)
test_location_val = '83.3595 -32.8601 0 1'
test_location2_val = '21.22474 -10.5601 50000 200'
self.assertNotEqual(d2x.call_args, None,
u'dict2xmlsubmission not called')
call_dict = d2x.call_args[0][0]
self.assertEqual(call_dict.get('test_location'), test_location_val,
u'Location prop test fail')
self.assertEqual(call_dict.get('test_location2'), test_location2_val,
u'Location2 prop test fail')
def test_reject_spaces_in_headers(self):
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv'),
'rb')
result = csv_import.submit_csv(self.user.username, self.xform,
non_utf8csv)
self.assertEqual(result.get('error'),
'CSV file fieldnames should not contain spaces',
'Incorrect error message returned.')
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv`
passing with the `request.FILES.get('csv_file')` upload for import.
"""
resp = submit_csv(request.user.username, self.get_object(),
request.FILES.get('csv_file'))
return Response(data=resp,
status=status.HTTP_200_OK if resp.get('error') is None
else status.HTTP_400_BAD_REQUEST)
def test_submit_csv_edits(self):
xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main",
"tests", "fixtures", "tutorial.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.get()
csv_import.submit_csv(self.user.username, self.xform, self.good_csv)
self.assertEqual(Instance.objects.count(), 9,
u'submit_csv edits #1 test Failed!')
edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv'))
edit_csv_str = edit_csv.read()
edit_csv = StringIO(
edit_csv_str.format(
* [x.get('uuid') for x in Instance.objects.values('uuid')]))
count = Instance.objects.count()
csv_import.submit_csv(self.user.username, self.xform, edit_csv)
self.assertEqual(Instance.objects.count(), count,
u'submit_csv edits #2 test Failed!')
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(
get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
elif csv_file.name.split('.')[-1] != CSV_EXTENSION:
resp.update({u'error': u'csv_file not a csv file'})
else:
overwrite = request.query_params.get('overwrite')
overwrite = True \
if overwrite and overwrite.lower() == 'true' else False
size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD
if csv_file.size < size_threshold:
resp.update(
submit_csv(request.user.username, self.object,
csv_file, overwrite))
else:
csv_file.seek(0)
upload_to = os.path.join(request.user.username,
'csv_imports', csv_file.name)
file_name = default_storage.save(upload_to, csv_file)
task = submit_csv_async.delay(request.user.username,
self.object.pk, file_name,
overwrite)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(data=resp,
status=status.HTTP_200_OK if resp.get('error') is None
else status.HTTP_400_BAD_REQUEST)
def test_submit_csv_instance_id_consistency(self, safe_create_instance):
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
safe_create_instance.return_value = {}
single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb')
csv_import.submit_csv(self.user.username, self.xform, single_csv)
xml_file_param = BytesIO(
open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read())
safe_create_args = list(safe_create_instance.call_args[0])
instance_xml = fromstring(safe_create_args[1].getvalue())
single_instance_xml = fromstring(xml_file_param.getvalue())
instance_id = [
m.find('instanceID').text for m in instance_xml.findall('meta')][0]
single_instance_id = [m.find('instanceID').text for m in
single_instance_xml.findall('meta')][0]
self.assertEqual(
len(instance_id), len(single_instance_id),
"Same uuid length in generated xml")
def test_submit_csv_instance_id_consistency(self, safe_create_instance):
self._publish_xls_file(self.xls_file_path)
self.xform = XForm.objects.get()
safe_create_instance.return_value = {}
single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb')
csv_import.submit_csv(self.user.username, self.xform, single_csv)
xml_file_param = BytesIO(
open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read())
safe_create_args = list(safe_create_instance.call_args[0])
instance_xml = fromstring(safe_create_args[1].getvalue())
single_instance_xml = fromstring(xml_file_param.getvalue())
instance_id = [
m.find('instanceID').text for m in instance_xml.findall('meta')][0]
single_instance_id = [m.find('instanceID').text for m in
single_instance_xml.findall('meta')][0]
self.assertEqual(
len(instance_id), len(single_instance_id),
"Same uuid length in generated xml")
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
elif csv_file.name.split('.')[-1] != CSV_EXTENSION:
resp.update({u'error': u'csv_file not a csv file'})
else:
overwrite = request.query_params.get('overwrite')
overwrite = True \
if overwrite and overwrite.lower() == 'true' else False
size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD
if csv_file.size < size_threshold:
resp.update(submit_csv(request.user.username,
self.object, csv_file, overwrite))
else:
csv_file.seek(0)
upload_to = os.path.join(request.user.username,
'csv_imports', csv_file.name)
file_name = default_storage.save(upload_to, csv_file)
task = submit_csv_async.delay(request.user.username,
self.object.pk, file_name,
overwrite)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv`
passing with the `request.FILES.get('csv_file')` upload for import.
"""
resp = submit_csv(request.user.username,
self.get_object(),
request.FILES.get('csv_file'))
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
def test_import_non_utf8_csv(self):
xls_file_path = os.path.join(self.fixtures_dir, "mali_health.xls")
self._publish_xls_file(xls_file_path)
self.xform = XForm.objects.get()
count = Instance.objects.count()
non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv'),
'rb')
result = csv_import.submit_csv(self.user.username, self.xform,
non_utf8_csv)
self.assertEqual(result.get('error'), 'CSV file must be utf-8 encoded',
'Incorrect error message returned.')
self.assertEqual(Instance.objects.count(), count,
'Non unicode csv import rollback failed!')
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(
get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
else:
num_rows = sum(1 for row in csv_file) - 1
if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD:
resp.update(
submit_csv(request.user.username, self.object,
csv_file))
else:
tmp_file_path = utils.generate_tmp_path(csv_file)
task = submit_csv_async.delay(request.user.username,
self.object, tmp_file_path)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(data=resp,
status=status.HTTP_200_OK if resp.get('error') is None
else status.HTTP_400_BAD_REQUEST)
def csv_import(self, request, *args, **kwargs):
"""
Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv`
passing with the `request.FILES.get('csv_file')` upload for import.
"""
xform = self.get_object()
if request.user != xform.user:
# Access control for this endpoint previously relied on testing
# that the user had `logger.add_xform` on this specific XForm,
# which is meaningless but does get assigned to the XForm owner by
# the post-save signal handler
# `onadata.apps.logger.models.xform.set_object_permissions()`.
# For safety and clarity, this endpoint now explicitly denies
# access to all non-owners.
raise PermissionDenied
resp = submit_csv(request, xform, request.FILES.get('csv_file'))
return Response(data=resp,
status=status.HTTP_200_OK if resp.get('error') is None
else status.HTTP_400_BAD_REQUEST)
def test_submit_csv_param_sanity_check(self):
resp = csv_import.submit_csv(self.request, self.xform, 123456)
self.assertIsNotNone(resp.get('error'))
def test_nested_geo_paths_csv(self):
good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv'))
csv_import.submit_csv(self.user.username, self.xform, good_csv)
self.assertEqual(Instance.objects.count(), 9,
u'submit_csv edits #1 test Failed!')
def test_submit_csv_param_sanity_check(self):
resp = csv_import.submit_csv(u'userX', XForm(), 123456)
self.assertIsNotNone(resp.get('error'))
def test_nested_geo_paths_csv(self):
good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv'))
csv_import.submit_csv(self.user.username, self.xform, good_csv)
self.assertEqual(Instance.objects.count(),
9, u'submit_csv edits #1 test Failed!')
def test_submit_csv_param_sanity_check(self):
resp = csv_import.submit_csv(u'userX', XForm(), 123456)
self.assertIsNotNone(resp.get('error'))
