def test_submission_xls_to_csv(self):
"""Test that submission_xls_to_csv converts to csv"""
c_csv_file = csv_import.submission_xls_to_csv(self.good_xls)
c_csv_file.seek(0)
c_csv_reader = ucsv.DictReader(c_csv_file, encoding='utf-8-sig')
g_csv_reader = ucsv.DictReader(self.good_csv, encoding='utf-8-sig')
self.assertEqual(g_csv_reader.fieldnames[10],
c_csv_reader.fieldnames[10])
def test_submission_xls_to_csv(self):
"""Test that submission_xls_to_csv converts to csv"""
c_csv_file = csv_import.submission_xls_to_csv(
self.good_xls)
c_csv_file.seek(0)
c_csv_reader = ucsv.DictReader(c_csv_file, encoding='utf-8-sig')
g_csv_reader = ucsv.DictReader(self.good_csv, encoding='utf-8-sig')
self.assertEqual(
g_csv_reader.fieldnames[10], c_csv_reader.fieldnames[10])
def data_import(self, request, *args, **kwargs):
""" Endpoint for CSV and XLS data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling and
:py:func:`onadata.libs.utils.csv_import.submission_xls_to_csv`
for POST request passing the `request.FILES.get('xls_file')` upload for
import if xls_file is provided instead of csv_file
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(
get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
xls_file = request.FILES.get('xls_file', None)
if csv_file is None and xls_file is None:
resp.update({u'error': u'csv_file and xls_file field empty'})
elif xls_file and \
xls_file.name.split('.')[-1] not in XLS_EXTENSIONS:
resp.update({u'error': u'xls_file not an excel file'})
elif csv_file and csv_file.name.split('.')[-1] != CSV_EXTENSION:
resp.update({u'error': u'csv_file not a csv file'})
else:
if xls_file and xls_file.name.split('.')[-1] in XLS_EXTENSIONS:
csv_file = submission_xls_to_csv(xls_file)
overwrite = request.query_params.get('overwrite')
overwrite = True \
if overwrite and overwrite.lower() == 'true' else False
size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD
try:
csv_size = csv_file.size
except AttributeError:
csv_size = csv_file.__sizeof__()
if csv_size < size_threshold:
resp.update(
submit_csv(request.user.username, self.object,
csv_file, overwrite))
else:
csv_file.seek(0)
upload_to = os.path.join(request.user.username,
'csv_imports', csv_file.name)
file_name = default_storage.save(upload_to, csv_file)
task = submit_csv_async.delay(request.user.username,
self.object.pk, file_name,
overwrite)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(data=resp,
status=status.HTTP_200_OK if resp.get('error') is None
else status.HTTP_400_BAD_REQUEST)
def data_import(self, request, *args, **kwargs):
""" Endpoint for CSV and XLS data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling and
:py:func:`onadata.libs.utils.csv_import.submission_xls_to_csv`
for POST request passing the `request.FILES.get('xls_file')` upload for
import if xls_file is provided instead of csv_file
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
xls_file = request.FILES.get('xls_file', None)
if csv_file is None and xls_file is None:
resp.update({u'error': u'csv_file and xls_file field empty'})
elif xls_file and \
xls_file.name.split('.')[-1] not in XLS_EXTENSIONS:
resp.update({u'error': u'xls_file not an excel file'})
elif csv_file and csv_file.name.split('.')[-1] != CSV_EXTENSION:
resp.update({u'error': u'csv_file not a csv file'})
else:
if xls_file and xls_file.name.split('.')[-1] in XLS_EXTENSIONS:
csv_file = submission_xls_to_csv(xls_file)
overwrite = request.query_params.get('overwrite')
overwrite = True \
if overwrite and overwrite.lower() == 'true' else False
size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD
try:
csv_size = csv_file.size
except AttributeError:
csv_size = csv_file.__sizeof__()
if csv_size < size_threshold:
resp.update(submit_csv(request.user.username,
self.object, csv_file, overwrite))
else:
csv_file.seek(0)
upload_to = os.path.join(request.user.username,
'csv_imports', csv_file.name)
file_name = default_storage.save(upload_to, csv_file)
task = submit_csv_async.delay(request.user.username,
self.object.pk, file_name,
overwrite)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
