def test_nested_geo_paths_csv(self): self.xls_file_path = os.path.join(self.fixtures_dir, "tutorial-nested-geo.xls") self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open(os.path.join(self.fixtures_dir, "another_good.csv")) csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 9, u"submit_csv edits #1 test Failed!")
def test_csv_with_multiple_select_in_one_column(self): self.xls_file_path = os.path.join(self.fixtures_dir, "form_with_multiple_select.xlsx") self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open(os.path.join(self.fixtures_dir, "csv_import_with_multiple_select.csv")) csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 1, u"submit_csv edits #1 test Failed!")
def test_submit_csv_and_rollback(self): count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), count + 9, u'submit_csv test Failed!') # Check that correct # of submissions belong to our user self.assertEqual(Instance.objects.filter(user=self.user).count(), count + 8, u'submit_csv username check failed!')
def test_excel_date_conversion(self): """Convert date from 01/01/1900 to 01-01-1900""" date_md_form = """ | survey | | | type | name | label | | | today | today | Today | | | text | name | Name | | | date | tdate | Date | | | start | start | | | | end | end | | | | dateTime | now | Now | | choices | | | list name | name | label | | settings | | | form_title | form_id | | | Dates | dates | """ self._create_user_and_login() data = {'name': 'data'} survey = self.md_to_pyxform_survey(date_md_form, kwargs=data) survey['sms_keyword'] = survey['id_string'] project = get_user_default_project(self.user) xform = XForm(created_by=self.user, user=self.user, xml=survey.to_xml(), json=survey.to_json(), project=project) xform.save() date_csv = open(os.path.join( self.fixtures_dir, 'date.csv'), 'rb') date_csv.seek(0) csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig') xl_dates = [] xl_datetime = [] # xl dates for row in csv_reader: xl_dates.append(row.get('tdate')) xl_datetime.append(row.get('now')) csv_import.submit_csv(self.user.username, xform, date_csv) # converted dates conv_dates = [instance.json.get('tdate') for instance in Instance.objects.filter( xform=xform).order_by('date_created')] conv_datetime = [instance.json.get('now') for instance in Instance.objects.filter( xform=xform).order_by('date_created')] self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019']) self.assertEqual( xl_datetime, [u'6/12/2020 13:20', u'2019-03-11T16:00:51.147+02:00']) self.assertEqual( conv_datetime, [u'2020-06-12T13:20:00.000000', u'2019-03-11T16:00:51.147+02:00']) self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26'])
def test_data_upload(self): """Data upload for submissions with no uuids""" self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() single_csv = open(os.path.join( self.fixtures_dir, 'single_data_upload.csv'), 'rb') csv_import.submit_csv(self.user.username, self.xform, single_csv) self.xform.refresh_from_db() self.assertEqual(self.xform.num_of_submissions, count + 1)
def test_nested_geo_paths_csv(self): self.xls_file_path = os.path.join(self.fixtures_dir, 'tutorial-nested-geo.xls') self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv'), 'rb') csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 9, 'submit_csv edits #1 test Failed!')
def test_submit_csv_and_rollback(self): xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), count + 9, u"submit_csv test Failed!") # Check that correct # of submissions belong to our user self.assertEqual( Instance.objects.filter(user=self.user).count(), count + 8, u"submit_csv username check failed!" )
def test_csv_with_multiple_select_in_one_column(self): self.xls_file_path = os.path.join(self.fixtures_dir, 'form_with_multiple_select.xlsx') self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() good_csv = open( os.path.join(self.fixtures_dir, 'csv_import_with_multiple_select.csv')) csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 1, u'submit_csv edits #1 test Failed!')
def test_data_upload(self, send_message_mock): """Data upload for submissions with no uuids""" self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() single_csv = open(os.path.join( self.fixtures_dir, 'single_data_upload.csv'), 'rb') csv_import.submit_csv(self.user.username, self.xform, single_csv) self.xform.refresh_from_db() self.assertEqual(self.xform.num_of_submissions, count + 1) # message sent upon submission creation self.assertTrue(send_message_mock.called) send_message_mock.called_with(self.xform.id, XFORM, SUBMISSION_CREATED)
def test_csv_with_repeats_import(self): self.xls_file_path = os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats.xls') repeats_csv = open( os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats.csv')) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() csv_import.submit_csv(self.user.username, self.xform, repeats_csv) count = self.xform.instances.count() self.assertEqual(count, 1 + pre_count)
def test_csv_with_repeats_import(self): self.xls_file_path = os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats.xls') repeats_csv = open( os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats.csv'), 'rb') self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() csv_import.submit_csv(self.user.username, self.xform, repeats_csv) count = self.xform.instances.count() self.assertEqual(count, 1 + pre_count)
def test_submit_csv_and_rollback(self): xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), count + 9, u'submit_csv test Failed!') # Check that correct # of submissions belong to our user self.assertEqual( Instance.objects.filter(user=self.user).count(), count + 8, u'submit_csv username check failed!')
def test_submit_csv_edits(self): csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), 9, 'submit_csv edits #1 test Failed!') edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv')) edit_csv_str = edit_csv.read() edit_csv = StringIO(edit_csv_str.format( *[x.get('uuid') for x in Instance.objects.values('uuid')])) count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, edit_csv) self.assertEqual(Instance.objects.count(), count, 'submit_csv edits #2 test Failed!')
def test_submit_csv_edits(self): csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), 9, u'submit_csv edits #1 test Failed!') edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv')) edit_csv_str = edit_csv.read() edit_csv = StringIO(edit_csv_str.format( *[x.get('uuid') for x in Instance.objects.values('uuid')])) count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, edit_csv) self.assertEqual(Instance.objects.count(), count, u'submit_csv edits #2 test Failed!')
def test_submit_csv_xml_location_property_test(self, d2x, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = [None] single_csv = open(os.path.join(self.fixtures_dir, "single.csv")) csv_import.submit_csv(self.user.username, self.xform, single_csv) test_location_val = "83.3595 -32.8601 0 1" test_location2_val = "21.22474 -10.5601 50000 200" self.assertNotEqual(d2x.call_args, None, u"dict2xmlsubmission not called") call_dict = d2x.call_args[0][0] self.assertEqual(call_dict.get("test_location"), test_location_val, u"Location prop test fail") self.assertEqual(call_dict.get("test_location2"), test_location2_val, u"Location2 prop test fail")
def test_submit_csv_edits(self): xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), 9, u"submit_csv edits #1 test Failed!") edit_csv = open(os.path.join(self.fixtures_dir, "edit.csv")) edit_csv_str = edit_csv.read() edit_csv = StringIO(edit_csv_str.format(*[x.get("uuid") for x in Instance.objects.values("uuid")])) count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, edit_csv) self.assertEqual(Instance.objects.count(), count, u"submit_csv edits #2 test Failed!")
def test_reject_spaces_in_headers(self): non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv')) result = csv_import.submit_csv(self.user.username, self.xform, non_utf8csv) self.assertEqual(result.get('error'), u'CSV file fieldnames should not contain spaces', u'Incorrect error message returned.')
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and :py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status for GET requests passing `job_uuid` query param for job progress polling """ resp = {} if request.method == 'GET': resp.update(get_async_csv_submission_status( request.QUERY_PARAMS.get('job_uuid'))) else: csv_file = request.FILES.get('csv_file', None) if csv_file is None: resp.update({u'error': u'csv_file field empty'}) else: num_rows = sum(1 for row in csv_file) - 1 if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD: resp.update(submit_csv(request.user.username, self.get_object(), csv_file)) else: resp.update( {u'task_id': submit_csv_async.delay( request.user.username, self.get_object(), csv_file).task_id}) return Response( data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and :py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status for GET requests passing `job_uuid` query param for job progress polling """ resp = {} if request.method == 'GET': resp.update( get_async_csv_submission_status( request.QUERY_PARAMS.get('job_uuid'))) else: csv_file = request.FILES.get('csv_file', None) if csv_file is None: resp.update({u'error': u'csv_file field empty'}) else: num_rows = sum(1 for row in csv_file) - 1 if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD: resp.update( submit_csv(request.user.username, self.get_object(), csv_file)) else: resp.update({ u'task_id': submit_csv_async.delay(request.user.username, self.get_object(), csv_file).task_id }) return Response(data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def test_enforces_data_type_and_rollback(self): """ Test that data type constraints are enforced and instances created during the process are rolled back """ # Test integer constraint is enforced xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.last() bad_data = open( os.path.join(self.fixtures_dir, 'bad_data.csv'), 'rb') count = Instance.objects.count() result = csv_import.submit_csv(self.user.username, self.xform, bad_data) expected_error = ( "Invalid CSV data imported in row(s): {1: ['Unknown date format(s)" ": 2014-0903', 'Unknown datetime format(s): sdsa', " "'Unknown integer format(s): 21.53'], 2: ['Unknown integer format" "(s): 22.32'], 4: ['Unknown date format(s): 2014-0900'], " "5: ['Unknown date format(s): 2014-0901'], 6: ['Unknown " "date format(s): 2014-0902'], 7: ['Unknown date format(s):" " 2014-0903']}") self.assertEqual( result.get('error'), expected_error) # Assert all created instances were rolled back self.assertEqual(count, Instance.objects.count())
def test_submit_csv_xml_location_property_test( self, d2x, safe_create_instance): safe_create_instance.return_value = [None, ] single_csv = open(os.path.join(self.fixtures_dir, 'single.csv')) csv_import.submit_csv(self.user.username, self.xform, single_csv) test_location_val = '83.3595 -32.8601 0 1' test_location2_val = '21.22474 -10.5601 50000 200' self.assertNotEqual(d2x.call_args, None, u'dict2xmlsubmission not called') call_dict = d2x.call_args[0][0] self.assertEqual(call_dict.get('test_location'), test_location_val, u'Location prop test fail') self.assertEqual(call_dict.get('test_location2'), test_location2_val, u'Location2 prop test fail')
def test_reject_spaces_in_headers(self): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() non_utf8csv = open(os.path.join(self.fixtures_dir, "header_space.csv")) result = csv_import.submit_csv(self.user.username, self.xform, non_utf8csv) self.assertEqual( result.get("error"), u"CSV file fieldnames should not contain spaces", u"Incorrect error message returned." )
def test_csv_with__more_than_4_repeats_import(self): self.xls_file_path = os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats.xls') repeats_csv = open( os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats_import.csv')) self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() csv_import.submit_csv(self.user.username, self.xform, repeats_csv) count = self.xform.instances.count() self.assertEqual(count, 1 + pre_count) instance = self.xform.instances.last() # repeats should be 6 self.assertEqual(6, len(instance.json.get('children')))
def test_submit_csv_xml_params(self, safe_create_instance): safe_create_instance.return_value = {} single_csv = open(os.path.join(self.fixtures_dir, 'single.csv')) csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = StringIO( open(os.path.join(self.fixtures_dir, 'single.xml')).read()) safe_create_args = list(safe_create_instance.call_args[0]) self.assertEqual(safe_create_args[0], self.user.username, u'Wrong username passed') self.assertEqual(strip_xml_uuid(safe_create_args[1].getvalue()), strip_xml_uuid(xml_file_param.getvalue()), u'Wrong xml param passed') self.assertEqual(safe_create_args[2], [], u'Wrong media array param passed') self.assertEqual(safe_create_args[3], self.xform.uuid, u'Wrong xform uuid passed') self.assertEqual(safe_create_args[4], None)
def test_excel_date_conversion(self): """Convert date from 01/01/1900 to 01-01-1900""" date_md_form = """ | survey | | | type | name | label | | | today | today | Today | | | text | name | Name | | | date | tdate | Date | | choices | | | list name | name | label | | settings | | | form_title | form_id | | | Dates | dates | """ self._create_user_and_login() data = {'name': 'data'} survey = self.md_to_pyxform_survey(date_md_form, kwargs=data) survey['sms_keyword'] = survey['id_string'] project = get_user_default_project(self.user) xform = XForm(created_by=self.user, user=self.user, xml=survey.to_xml(), json=survey.to_json(), project=project) xform.save() date_csv = open(os.path.join(self.fixtures_dir, 'date.csv'), 'rb') date_csv.seek(0) csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig') xl_dates = [] # xl dates for row in csv_reader: xl_dates.append(row.get('tdate')) csv_import.submit_csv(self.user.username, xform, date_csv) # converted dates conv_dates = [ instance.json.get('tdate') for instance in Instance.objects.filter( xform=xform).order_by('date_created') ] self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019']) self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26'])
def test_import_non_utf8_csv(self): count = Instance.objects.count() non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv')) result = csv_import.submit_csv(self.user.username, self.xform, non_utf8_csv) self.assertEqual(result.get('error'), u'CSV file must be utf-8 encoded', u'Incorrect error message returned.') self.assertEqual(Instance.objects.count(), count, u'Non unicode csv import rollback failed!')
def test_csv_with__more_than_4_repeats_import(self): self.xls_file_path = os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats.xls') repeats_csv = open( os.path.join(self.this_directory, 'fixtures', 'csv_export', 'tutorial_w_repeats_import.csv'), 'rb') self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() pre_count = self.xform.instances.count() csv_import.submit_csv(self.user.username, self.xform, repeats_csv) count = self.xform.instances.count() self.assertEqual(count, 1 + pre_count) instance = self.xform.instances.last() # repeats should be 6 self.assertEqual(6, len(instance.json.get('children')))
def test_submit_csv_xml_params(self, safe_create_instance): safe_create_instance.return_value = {} single_csv = open(os.path.join(self.fixtures_dir, 'single.csv')) csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = StringIO(open(os.path.join(self.fixtures_dir, 'single.xml')).read()) safe_create_args = list(safe_create_instance.call_args[0]) self.assertEqual(safe_create_args[0], self.user.username, u'Wrong username passed') self.assertEqual(strip_xml_uuid(safe_create_args[1].getvalue()), strip_xml_uuid(xml_file_param.getvalue()), u'Wrong xml param passed') self.assertEqual(safe_create_args[2], [], u'Wrong media array param passed') self.assertEqual(safe_create_args[3], self.xform.uuid, u'Wrong xform uuid passed') self.assertEqual(safe_create_args[4], None)
def test_import_non_utf8_csv(self): xls_file_path = os.path.join(self.fixtures_dir, "mali_health.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() non_utf8_csv = open(os.path.join(self.fixtures_dir, "non_utf8.csv")) result = csv_import.submit_csv(self.user.username, self.xform, non_utf8_csv) self.assertEqual(result.get("error"), u"CSV file must be utf-8 encoded", u"Incorrect error message returned.") self.assertEqual(Instance.objects.count(), count, u"Non unicode csv import rollback failed!")
def test_submit_csv_xml_location_property_test(self, d2x, safe_create_instance): safe_create_instance.return_value = [ None, ] single_csv = open(os.path.join(self.fixtures_dir, 'single.csv')) csv_import.submit_csv(self.user.username, self.xform, single_csv) test_location_val = '83.3595 -32.8601 0 1' test_location2_val = '21.22474 -10.5601 50000 200' self.assertNotEqual(d2x.call_args, None, u'dict2xmlsubmission not called') call_dict = d2x.call_args[0][0] self.assertEqual(call_dict.get('test_location'), test_location_val, u'Location prop test fail') self.assertEqual(call_dict.get('test_location2'), test_location2_val, u'Location2 prop test fail')
def test_reject_spaces_in_headers(self): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() non_utf8csv = open(os.path.join(self.fixtures_dir, 'header_space.csv'), 'rb') result = csv_import.submit_csv(self.user.username, self.xform, non_utf8csv) self.assertEqual(result.get('error'), 'CSV file fieldnames should not contain spaces', 'Incorrect error message returned.')
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` passing with the `request.FILES.get('csv_file')` upload for import. """ resp = submit_csv(request.user.username, self.get_object(), request.FILES.get('csv_file')) return Response(data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def test_submit_csv_edits(self): xls_file_path = os.path.join(settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() csv_import.submit_csv(self.user.username, self.xform, self.good_csv) self.assertEqual(Instance.objects.count(), 9, u'submit_csv edits #1 test Failed!') edit_csv = open(os.path.join(self.fixtures_dir, 'edit.csv')) edit_csv_str = edit_csv.read() edit_csv = StringIO( edit_csv_str.format( * [x.get('uuid') for x in Instance.objects.values('uuid')])) count = Instance.objects.count() csv_import.submit_csv(self.user.username, self.xform, edit_csv) self.assertEqual(Instance.objects.count(), count, u'submit_csv edits #2 test Failed!')
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and :py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status for GET requests passing `job_uuid` query param for job progress polling """ self.object = self.get_object() resp = {} if request.method == 'GET': try: resp.update( get_async_csv_submission_status( request.query_params.get('job_uuid'))) self.last_modified_date = timezone.now() except ValueError: raise ParseError(('The instance of the result is not a ' 'basestring; the job_uuid variable might ' 'be incorrect')) else: csv_file = request.FILES.get('csv_file', None) if csv_file is None: resp.update({u'error': u'csv_file field empty'}) elif csv_file.name.split('.')[-1] != CSV_EXTENSION: resp.update({u'error': u'csv_file not a csv file'}) else: overwrite = request.query_params.get('overwrite') overwrite = True \ if overwrite and overwrite.lower() == 'true' else False size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD if csv_file.size < size_threshold: resp.update( submit_csv(request.user.username, self.object, csv_file, overwrite)) else: csv_file.seek(0) upload_to = os.path.join(request.user.username, 'csv_imports', csv_file.name) file_name = default_storage.save(upload_to, csv_file) task = submit_csv_async.delay(request.user.username, self.object.pk, file_name, overwrite) if task is None: raise ParseError('Task not found') else: resp.update({u'task_id': task.task_id}) return Response(data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def test_submit_csv_instance_id_consistency(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() safe_create_instance.return_value = {} single_csv = open(os.path.join(self.fixtures_dir, 'single.csv'), 'rb') csv_import.submit_csv(self.user.username, self.xform, single_csv) xml_file_param = BytesIO( open(os.path.join(self.fixtures_dir, 'single.xml'), 'rb').read()) safe_create_args = list(safe_create_instance.call_args[0]) instance_xml = fromstring(safe_create_args[1].getvalue()) single_instance_xml = fromstring(xml_file_param.getvalue()) instance_id = [ m.find('instanceID').text for m in instance_xml.findall('meta')][0] single_instance_id = [m.find('instanceID').text for m in single_instance_xml.findall('meta')][0] self.assertEqual( len(instance_id), len(single_instance_id), "Same uuid length in generated xml")
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and :py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status for GET requests passing `job_uuid` query param for job progress polling """ self.object = self.get_object() resp = {} if request.method == 'GET': try: resp.update(get_async_csv_submission_status( request.query_params.get('job_uuid'))) self.last_modified_date = timezone.now() except ValueError: raise ParseError(('The instance of the result is not a ' 'basestring; the job_uuid variable might ' 'be incorrect')) else: csv_file = request.FILES.get('csv_file', None) if csv_file is None: resp.update({u'error': u'csv_file field empty'}) elif csv_file.name.split('.')[-1] != CSV_EXTENSION: resp.update({u'error': u'csv_file not a csv file'}) else: overwrite = request.query_params.get('overwrite') overwrite = True \ if overwrite and overwrite.lower() == 'true' else False size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD if csv_file.size < size_threshold: resp.update(submit_csv(request.user.username, self.object, csv_file, overwrite)) else: csv_file.seek(0) upload_to = os.path.join(request.user.username, 'csv_imports', csv_file.name) file_name = default_storage.save(upload_to, csv_file) task = submit_csv_async.delay(request.user.username, self.object.pk, file_name, overwrite) if task is None: raise ParseError('Task not found') else: resp.update({u'task_id': task.task_id}) return Response( data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` passing with the `request.FILES.get('csv_file')` upload for import. """ resp = submit_csv(request.user.username, self.get_object(), request.FILES.get('csv_file')) return Response( data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def test_import_non_utf8_csv(self): xls_file_path = os.path.join(self.fixtures_dir, "mali_health.xls") self._publish_xls_file(xls_file_path) self.xform = XForm.objects.get() count = Instance.objects.count() non_utf8_csv = open(os.path.join(self.fixtures_dir, 'non_utf8.csv'), 'rb') result = csv_import.submit_csv(self.user.username, self.xform, non_utf8_csv) self.assertEqual(result.get('error'), 'CSV file must be utf-8 encoded', 'Incorrect error message returned.') self.assertEqual(Instance.objects.count(), count, 'Non unicode csv import rollback failed!')
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST requests passing the `request.FILES.get('csv_file')` upload for import and :py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status for GET requests passing `job_uuid` query param for job progress polling """ self.object = self.get_object() resp = {} if request.method == 'GET': try: resp.update( get_async_csv_submission_status( request.query_params.get('job_uuid'))) self.last_modified_date = timezone.now() except ValueError: raise ParseError(('The instance of the result is not a ' 'basestring; the job_uuid variable might ' 'be incorrect')) else: csv_file = request.FILES.get('csv_file', None) if csv_file is None: resp.update({u'error': u'csv_file field empty'}) else: num_rows = sum(1 for row in csv_file) - 1 if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD: resp.update( submit_csv(request.user.username, self.object, csv_file)) else: tmp_file_path = utils.generate_tmp_path(csv_file) task = submit_csv_async.delay(request.user.username, self.object, tmp_file_path) if task is None: raise ParseError('Task not found') else: resp.update({u'task_id': task.task_id}) return Response(data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def csv_import(self, request, *args, **kwargs): """ Endpoint for CSV data imports Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` passing with the `request.FILES.get('csv_file')` upload for import. """ xform = self.get_object() if request.user != xform.user: # Access control for this endpoint previously relied on testing # that the user had `logger.add_xform` on this specific XForm, # which is meaningless but does get assigned to the XForm owner by # the post-save signal handler # `onadata.apps.logger.models.xform.set_object_permissions()`. # For safety and clarity, this endpoint now explicitly denies # access to all non-owners. raise PermissionDenied resp = submit_csv(request, xform, request.FILES.get('csv_file')) return Response(data=resp, status=status.HTTP_200_OK if resp.get('error') is None else status.HTTP_400_BAD_REQUEST)
def test_submit_csv_param_sanity_check(self): resp = csv_import.submit_csv(self.request, self.xform, 123456) self.assertIsNotNone(resp.get('error'))
def test_nested_geo_paths_csv(self): good_csv = open(os.path.join(self.fixtures_dir, 'another_good.csv')) csv_import.submit_csv(self.user.username, self.xform, good_csv) self.assertEqual(Instance.objects.count(), 9, u'submit_csv edits #1 test Failed!')
def test_submit_csv_param_sanity_check(self): resp = csv_import.submit_csv(u'userX', XForm(), 123456) self.assertIsNotNone(resp.get('error'))