def test_xform_has_uncommon_reference(self): """ Test creating a merged dataset that has matching fields but with uncommon reference variable. """ view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown(REFERENCE_ISSUE, self.user, id_string='b') data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform2.pk, "http://testserver/api/v1/forms/%s" % xform1.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) error_message = ( "There has been a problem trying to replace ${tunda} with the " "XPath to the survey element named 'tunda'. There is no survey " "element with this name.") self.assertIn('xforms', response.data) self.assertIn(error_message, response.data['xforms'])
def test_xform_has_uncommon_reference(self): """ Test creating a merged dataset that has matching fields but with uncommon reference variable. """ view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown( REFERENCE_ISSUE, self.user, id_string='b') data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform2.pk, "http://testserver/api/v1/forms/%s" % xform1.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) error_message = ( "There has been a problem trying to replace ${tunda} with the " "XPath to the survey element named 'tunda'. There is no survey " "element with this name.") self.assertIn('xforms', response.data) self.assertIn(error_message, response.data['xforms'])
def test_md_has_deleted_xforms(self): """ Test creating a merged dataset that includes a soft deleted form. """ view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown(MD, self.user, id_string='b') xform2.soft_delete() data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual(response.data, { 'xforms': [u'Invalid hyperlink - Object does not exist.'] })
def test_md_has_deleted_xforms(self): """ Test creating a merged dataset that includes a soft deleted form. """ view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown(MD, self.user, id_string='b') xform2.soft_delete() data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual( response.data, {'xforms': [u'Invalid hyperlink - Object does not exist.']})
def test_md_has_no_matching_fields(self): """ Test creating a merged dataset that has no matching fields. """ view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown(NOT_MATCHING, self.user, id_string='b') data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) self.assertEqual(response.data, {'xforms': [u'No matching fields in xforms.']})
def setUp(self): self.user = User.objects.create(username="******", email="*****@*****.**") self.project = UserProfile.objects.create(user=self.user) self.user.set_password("pass") self.project = get_user_default_project(self.user) self.xform1 = DataDictionary() self.xform1.user = self.user self.xform1.project = self.project self.xform1.json = '{"id_string": "yes_or_no_form", "children": '\ '[{"name": '\ '"yesno", "label": "Yes or no?", "type": "text"}],'\ ' "name": "yes_or_no", "title": "yes_or_no", "type'\ '": "survey"}'.strip() self.xform2 = DataDictionary() self.xform2.user = self.user self.xform2.project = self.project self.xform2.json = '{"id_string": "start_time_form", "children": '\ '[{"name":'\ '"start_time", "type": "start"}], "name": "start_t'\ 'ime_form", "title": "start_time_form",'\ '"type": "survey"}'\ .strip() self._get_xml_for_form(self.xform1) self._get_xml_for_form(self.xform2)
def test_get_merged_xform_survey(self): """ Test get_merged_xform_survey() """ with self.assertRaises(AssertionError): get_merged_xform_survey([]) self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(A_MD, self.user, id_string='a') xform2 = self._publish_markdown(B_MD, self.user, id_string='b') xform3 = self._publish_markdown(MD, self.user, id_string='c') expected = { u'name': u'data', u'title': u'pyxform_autotesttitle', u'sms_keyword': u'a', u'default_language': u'default', u'id_string': u'a', u'type': u'survey', u'children': [{ u'name': u'name', u'label': u'Name', u'type': u'text' }, { u'children': [{ u'name': u'female', u'label': u'Female' }, { u'name': u'male', u'label': u'Male' }], u'name': u'gender', u'label': u'Sex', u'type': u'select one' }, { u'control': { u'bodyless': True }, u'children': [{ u'name': u'instanceID', u'bind': { u'readonly': u'true()', u'calculate': u"concat('uuid:', uuid())" }, u'type': u'calculate' }], u'name': u'meta', u'type': u'group' }] } # yapf: disable with self.assertRaises(AssertionError): get_merged_xform_survey([xform1]) survey = get_merged_xform_survey([xform1, xform2]) self.assertEqual(survey.to_json_dict(), expected) # no matching fields with self.assertRaises(serializers.ValidationError): survey = get_merged_xform_survey([xform1, xform3])
def test_excel_date_conversion(self): """Convert date from 01/01/1900 to 01-01-1900""" date_md_form = """ | survey | | | type | name | label | | | today | today | Today | | | text | name | Name | | | date | tdate | Date | | | start | start | | | | end | end | | | | dateTime | now | Now | | choices | | | list name | name | label | | settings | | | form_title | form_id | | | Dates | dates | """ self._create_user_and_login() data = {'name': 'data'} survey = self.md_to_pyxform_survey(date_md_form, kwargs=data) survey['sms_keyword'] = survey['id_string'] project = get_user_default_project(self.user) xform = XForm(created_by=self.user, user=self.user, xml=survey.to_xml(), json=survey.to_json(), project=project) xform.save() date_csv = open(os.path.join(self.fixtures_dir, 'date.csv'), 'rb') date_csv.seek(0) csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig') xl_dates = [] xl_datetime = [] # xl dates for row in csv_reader: xl_dates.append(row.get('tdate')) xl_datetime.append(row.get('now')) csv_import.submit_csv(self.user.username, xform, date_csv) # converted dates conv_dates = [ instance.json.get('tdate') for instance in Instance.objects.filter( xform=xform).order_by('date_created') ] conv_datetime = [ instance.json.get('now') for instance in Instance.objects.filter( xform=xform).order_by('date_created') ] self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019']) self.assertEqual( xl_datetime, [u'6/12/2020 13:20', u'2019-03-11T16:00:51.147+02:00']) self.assertEqual( conv_datetime, [u'2020-06-12T13:20:00.000000', u'2019-03-11T16:00:51.147+02:00']) self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26'])
def test_excel_date_conversion(self): """Convert date from 01/01/1900 to 01-01-1900""" date_md_form = """ | survey | | | type | name | label | | | today | today | Today | | | text | name | Name | | | date | tdate | Date | | | start | start | | | | end | end | | | | dateTime | now | Now | | choices | | | list name | name | label | | settings | | | form_title | form_id | | | Dates | dates | """ self._create_user_and_login() data = {'name': 'data'} survey = self.md_to_pyxform_survey(date_md_form, kwargs=data) survey['sms_keyword'] = survey['id_string'] project = get_user_default_project(self.user) xform = XForm(created_by=self.user, user=self.user, xml=survey.to_xml(), json=survey.to_json(), project=project) xform.save() date_csv = open(os.path.join( self.fixtures_dir, 'date.csv'), 'rb') date_csv.seek(0) csv_reader = ucsv.DictReader(date_csv, encoding='utf-8-sig') xl_dates = [] xl_datetime = [] # xl dates for row in csv_reader: xl_dates.append(row.get('tdate')) xl_datetime.append(row.get('now')) csv_import.submit_csv(self.user.username, xform, date_csv) # converted dates conv_dates = [instance.json.get('tdate') for instance in Instance.objects.filter( xform=xform).order_by('date_created')] conv_datetime = [instance.json.get('now') for instance in Instance.objects.filter( xform=xform).order_by('date_created')] self.assertEqual(xl_dates, ['3/1/2019', '2/26/2019']) self.assertEqual( xl_datetime, [u'6/12/2020 13:20', u'2019-03-11T16:00:51.147+02:00']) self.assertEqual( conv_datetime, [u'2020-06-12T13:20:00.000000', u'2019-03-11T16:00:51.147+02:00']) self.assertEqual(conv_dates, ['2019-03-01', '2019-02-26'])
def test_correct_merged_dataset_data_for_charts(self): """Return correct data from the charts endpoint""" view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform_a = self._publish_markdown(MD, self.user, id_string='a') xform_b = self._publish_markdown(MD2, self.user, id_string='b') data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform_a.pk, "http://testserver/api/v1/forms/%s" % xform_b.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } # anonymous user request = self.factory.post('/', data=data) response = view(request) self.assertEqual(response.status_code, 401) request = self.factory.post('/', data=data) force_authenticate(request, user=self.user) response = view(request) self.assertEqual(response.status_code, 201) # make submission to form a xml = '<data id="b"><fruits>orange mango</fruits></data>' Instance(xform=xform_a, xml=xml).save() # make submission to form b xml = '<data id="b"><fruits>apple cherries</fruits></data>' Instance(xform=xform_b, xml=xml).save() data = {'field_xpath': 'fruits'} request = self.factory.get('/charts', data=data) force_authenticate(request, user=self.user) response = self.view(request, pk=response.data['id']) self.assertEqual(response.status_code, 200) # check that the data is correct expected_data = [{ 'fruits': ['Apple', 'Cherries'], 'count': 1 }, { 'fruits': ['Orange', 'Mango'], 'count': 1 }] self.assertEqual(response.data['data'], expected_data) # Ensure response is renderable response.render() cache.clear()
def publish(self, user, id_string=None, created_by=None): if self.is_valid(): # If a text (csv) representation of the xlsform is present, # this will save the file and pass it instead of the 'xls_file' # field. if 'text_xls_form' in self.cleaned_data\ and self.cleaned_data['text_xls_form'].strip(): csv_data = self.cleaned_data['text_xls_form'] # assigning the filename to a random string (quick fix) import random rand_name = "uploaded_form_%s.csv" % ''.join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) cleaned_xls_file = \ default_storage.save( upload_to(None, rand_name, user.username), ContentFile(csv_data)) else: cleaned_xls_file = self.cleaned_data['xls_file'] if not cleaned_xls_file: cleaned_url = (self.cleaned_data['xls_url'].strip() or self.cleaned_data['dropbox_xls_url'] or self.cleaned_data['csv_url']) cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = \ '_'.join(cleaned_xls_file.path.split('/')[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS: r = requests.get(cleaned_url) if r.headers.get('content-type') in \ VALID_XLSFORM_CONTENT_TYPES and \ r.status_code < 400: cleaned_xls_file = get_filename(r) cleaned_xls_file = \ upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urllib2.urlopen(cleaned_url).read()) cleaned_xls_file = \ default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data['project'] if project is None: project = get_user_default_project(user) else: project = self._project # publish the xls return publish_xls_form(cleaned_xls_file, user, project, id_string, created_by or user)
def _publish_markdown(self, md, user, project=None, **kwargs): kwargs['name'] = 'data' survey = self.md_to_pyxform_survey(md, kwargs=kwargs) survey['sms_keyword'] = survey['id_string'] if not project or not hasattr(self, 'project'): project = get_user_default_project(user) xform = DataDictionary(created_by=user, user=user, xml=survey.to_xml(), json=survey.to_json(), project=project) xform.save() return xform
def _publish_md(self, md, user, project=None): survey = self.md_to_pyxform_survey(md) if not project: project = get_user_default_project(user) xform = DataDictionary(created_by=user, user=user, xml=survey.to_xml(), json=survey.to_json(), project=project) xform.save() return xform
def save(self, **kwargs): user = User.objects.get(username=self.username) project = self.project or get_user_default_project(user) xls_file_path = upload_to( None, '%s%s.xls' % (self.xform.id_string, XForm.CLONED_SUFFIX), self.username) xls_data = default_storage.open(self.xform.xls.name) xls_file = default_storage.save(xls_file_path, xls_data) self.cloned_form = DataDictionary.objects.create(user=user, created_by=user, xls=xls_file, project=project)
def handle(self, *args, **options): try: xls_filepath = options['xls_filepath'] except KeyError: raise CommandError(_("You must provide the path to the xls file.")) # make sure path exists if not os.path.exists(xls_filepath): raise CommandError( _("The xls file '%s' does not exist.") % xls_filepath) try: username = options['username'] except KeyError: raise CommandError( _("You must provide the username to publish the form to.")) # make sure user exists try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError(_("The user '%s' does not exist.") % username) # wasteful but we need to get the id_string beforehand survey = create_survey_from_xls(xls_filepath) # check if a form with this id_string exists for this user form_already_exists = XForm.objects.filter( user=user, id_string=survey.id_string).count() > 0 # id_string of form to replace, if any id_string = None if form_already_exists: if 'replace' in options and options['replace']: id_string = survey.id_string self.stdout.write(_("Form already exist, replacing ..\n")) else: raise CommandError( _("The form with id_string '%s' already exists, use the -r" " option to replace it.") % survey.id_string) else: self.stdout.write(_("Form does NOT exist, publishing ..\n")) try: project_name = options['project_name'] project = Project.objects.get(name=project_name) except (KeyError, Project.DoesNotExist): project = get_user_default_project(user) # publish xls_file = django_file(xls_filepath, 'xls_file', 'application/vnd.ms-excel') publish_xls_form(xls_file, user, project, id_string) self.stdout.write(_("Done..\n"))
def test_retrieve_responses_merged_dataset(self): """ Test that a user is able to retrieve FLOIP Responses for Merged XForms """ MD = """ | survey | | | type | name | label | | | photo | image1 | Photo | """ # Create Merged XForm merged_dataset_view = MergedXFormViewSet.as_view({ 'post': 'create', }) project = get_user_default_project(self.user) self._publish_xls_form_to_project() self._make_submissions() xform = self._publish_markdown(MD, self.user, id_string='a') data = { 'xforms': [ f'http://testserver/api/v1/forms/{self.xform.pk}', f'http://testserver/api/v1/forms/{xform.pk}', ], 'name': 'Merged Dataset', 'project': f'http://testserver/api/v1/projects/{project.pk}', } request = self.factory.post('/', data=data, **self.extra) response = merged_dataset_view(request) self.assertEqual(response.status_code, 201) dataset_uuid = response.data['uuid'] # Assert that it's possible to retrieve the responses view = FloipViewSet.as_view({'get': 'responses'}) request = self.factory.get( f'/flow-results/packages/{dataset_uuid}/responses', content_type='application/vnd.api+json', **self.extra) response = view(request, uuid=dataset_uuid) self.assertEqual(response.status_code, 200) # Convert the returned generator object into a list response.data['attributes']['responses'] = list( response.data['attributes']['responses']) # The transportation form(self.xform) contains 11 responses # Assert that the responses are returned self.assertEqual(len(response.data['attributes']['responses']), 11)
def _publish_xls_file(self, path): if not path.startswith('/%s/' % self.user.username): path = os.path.join(self.this_directory, path) with open(path) as f: xls_file = InMemoryUploadedFile( f, 'xls_file', os.path.abspath(os.path.basename(path)), 'application/vnd.ms-excel', os.path.getsize(path), None) if not hasattr(self, 'project'): self.project = get_user_default_project(self.user) DataDictionary.objects.create(created_by=self.user, user=self.user, xls=xls_file, project=self.project)
def save(self, **kwargs): user = User.objects.get(username=self.username) project = get_user_default_project(user) xls_file_path = upload_to(None, '%s%s.xls' % ( self.xform.id_string, XForm.CLONED_SUFFIX), self.username) xls_data = default_storage.open(self.xform.xls.name) xls_file = default_storage.save(xls_file_path, xls_data) self.cloned_form = DataDictionary.objects.create( user=user, xls=xls_file, project=project )
def _create_merged_dataset(self, geo=False): view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown(MD, self.user, id_string='b') if geo: xform2.instances_with_geopoints = True xform2.save(update_fields=['instances_with_geopoints']) data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } # anonymous user request = self.factory.post('/', data=data) response = view(request) self.assertEqual(response.status_code, 401) request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 201) self.assertIn('id', response.data) self.assertIn('title', response.data) self.assertIn('xforms', response.data) expected_xforms_data = { 'id': xform1.pk, 'title': xform1.title, 'id_string': xform1.id_string, 'url': "http://testserver/api/v1/forms/%s" % xform1.pk, 'num_of_submissions': xform1.num_of_submissions, 'owner': xform1.user.username, 'project_id': self.project.pk, 'project_name': self.project.name } self.assertEqual(response.data['xforms'][0], expected_xforms_data) self.assertIsNotNone(response.data['uuid']) self.assertEqual(len(response.data['uuid']), 32) return response.data
def _contributions_form_submissions(self): count = XForm.objects.count() path = os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'forms', 'contributions') form_path = os.path.join(path, 'contributions.xml') with open(form_path, encoding='utf-8') as f: xml_file = ContentFile(f.read()) xml_file.name = 'contributions.xml' project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) instances_path = os.path.join(path, 'instances') for uuid in os.listdir(instances_path): s_path = os.path.join(instances_path, uuid, 'submission.xml') create_instance(self.user.username, open(s_path, 'rb'), []) self.assertEqual(self.xform.instances.count(), 6)
def test_merged_dataset_dict_contains_no_bind_attributes(self): """ Test get_merged_xform_survey(): should not contain bind elements. """ self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(GROUP_A_MD, self.user, id_string='a') xform2 = self._publish_markdown(GROUP_B_MD, self.user, id_string='b') xform3 = self._publish_markdown(GROUP_C_MD, self.user, id_string='c') survey = get_merged_xform_survey([xform1, xform2, xform3]) result = survey.to_json_dict() count = len([child for child in result["children"] if 'bind' in child]) # check that no elements within the newly created # merged_dataset_dict contains bind attributes self.assertEqual(count, 0)
def publish(self, user, id_string=None, created_by=None): if self.is_valid(): # If a text (csv) representation of the xlsform is present, # this will save the file and pass it instead of the 'xls_file' # field. if 'text_xls_form' in self.cleaned_data\ and self.cleaned_data['text_xls_form'].strip(): csv_data = self.cleaned_data['text_xls_form'] # assigning the filename to a random string (quick fix) import random rand_name = "uploaded_form_%s.csv" % ''.join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) cleaned_xls_file = \ default_storage.save( upload_to(None, rand_name, user.username), ContentFile(csv_data)) else: cleaned_xls_file = self.cleaned_data['xls_file'] if not cleaned_xls_file: cleaned_url = self.cleaned_data['xls_url'] if cleaned_url.strip() == u'': cleaned_url = self.cleaned_data['dropbox_xls_url'] cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = \ '_'.join(cleaned_xls_file.path.split('/')[-2:]) if cleaned_xls_file[-4:] != '.xls': cleaned_xls_file += '.xls' cleaned_xls_file = \ upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urllib2.urlopen(cleaned_url).read()) cleaned_xls_file = \ default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data['project'] if project is None: project = get_user_default_project(user) else: project = self._project # publish the xls return publish_xls_form(cleaned_xls_file, user, project, id_string, created_by or user)
def test_publish_xml_xlsform_download(self): count = XForm.objects.count() path = os.path.join( self.this_directory, '..', '..', 'api', 'tests', 'fixtures', 'forms', 'contributions', 'contributions.xml') f = open(path) xml_file = ContentFile(f.read()) f.close() xml_file.name = 'contributions.xml' project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) response = self.client.get(reverse(download_xlsform, kwargs={ 'username': self.user.username, 'id_string': 'contributions' }), follow=True) self.assertContains(response, 'No XLS file for your form ')
def _create_merged_dataset(self, geo=False): view = MergedXFormViewSet.as_view({ 'post': 'create', }) # pylint: disable=attribute-defined-outside-init self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') xform2 = self._publish_markdown(MD, self.user, id_string='b') if geo: xform2.instances_with_geopoints = True xform2.save(update_fields=['instances_with_geopoints']) data = { 'xforms': [ "http://testserver/api/v1/forms/%s" % xform1.pk, "http://testserver/api/v1/forms/%s" % xform2.pk, ], 'name': 'Merged Dataset', 'project': "http://testserver/api/v1/projects/%s" % self.project.pk, } # anonymous user request = self.factory.post('/', data=data) response = view(request) self.assertEqual(response.status_code, 401) request = self.factory.post('/', data=data, **self.extra) response = view(request) self.assertEqual(response.status_code, 201) self.assertIn('id', response.data) self.assertIn('title', response.data) self.assertIn('xforms', response.data) expected_xforms_data = { 'id': xform1.pk, 'title': xform1.title, 'id_string': xform1.id_string, 'url': "http://testserver/api/v1/forms/%s" % xform1.pk, 'num_of_submissions': xform1.num_of_submissions, 'owner': xform1.user.username, 'project_id': self.project.pk, 'project_name': self.project.name } self.assertEqual(response.data['xforms'][0], expected_xforms_data) return response.data
def _publish_xls_file(self, path): if not path.startswith('/%s/' % self.user.username): path = os.path.join(self.this_directory, path) with open(path) as f: xls_file = InMemoryUploadedFile(f, 'xls_file', os.path.basename(path), 'application/vnd.ms-excel', os.path.getsize(path), None) if not hasattr(self, 'project'): self.project = get_user_default_project(self.user) DataDictionary.objects.create( created_by=self.user, user=self.user, xls=xls_file, project=self.project )
def _contributions_form_submissions(self): count = XForm.objects.count() path = os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'forms', 'contributions') form_path = os.path.join(path, 'contributions.xml') f = open(form_path) xml_file = ContentFile(f.read()) f.close() xml_file.name = 'contributions.xml' project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) instances_path = os.path.join(path, 'instances') for uuid in os.listdir(instances_path): s_path = os.path.join(instances_path, uuid, 'submission.xml') create_instance(self.user.username, open(s_path), []) self.assertEqual(self.xform.instances.count(), 6)
def test_publish_xml_xlsform_download(self): count = XForm.objects.count() path = os.path.join( self.this_directory, "..", "..", "api", "tests", "fixtures", "forms", "contributions", "contributions.xml" ) f = open(path) xml_file = ContentFile(f.read()) f.close() xml_file.name = "contributions.xml" project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) response = self.client.get( reverse(download_xlsform, kwargs={"username": self.user.username, "id_string": "contributions"}), follow=True, ) self.assertContains(response, "No XLS file for your form ")
def publish(self, user, id_string=None, created_by=None): if self.is_valid(): # If a text (csv) representation of the xlsform is present, # this will save the file and pass it instead of the 'xls_file' # field. if "text_xls_form" in self.cleaned_data and self.cleaned_data["text_xls_form"].strip(): csv_data = self.cleaned_data["text_xls_form"] # assigning the filename to a random string (quick fix) import random rand_name = "uploaded_form_%s.csv" % "".join(random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) cleaned_xls_file = default_storage.save( upload_to(None, rand_name, user.username), ContentFile(csv_data) ) else: cleaned_xls_file = self.cleaned_data["xls_file"] if not cleaned_xls_file: cleaned_url = ( self.cleaned_data["xls_url"].strip() or self.cleaned_data["dropbox_xls_url"] or self.cleaned_data["csv_url"] ) cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = "_".join(cleaned_xls_file.path.split("/")[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in [".xls", ".xlsx", ".csv"]: cleaned_xls_file += ".xls" cleaned_xls_file = upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urllib2.urlopen(cleaned_url).read()) cleaned_xls_file = default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data["project"] if project is None: project = get_user_default_project(user) else: project = self._project # publish the xls return publish_xls_form(cleaned_xls_file, user, project, id_string, created_by or user)
def setUp(self): self.user = User.objects.create(username="******") profile, c = UserProfile.objects.get_or_create(user=self.user) profile.require_auth = False profile.save() self.project = get_user_default_project(self.user) absolute_path = get_absolute_path("forms") open_forms = open_all_files(absolute_path) self.json = '{"default_language": "default", ' \ '"id_string": "Water_2011_03_17", "children": [], ' \ '"name": "Water_2011_03_17", ' \ '"title": "Water_2011_03_17", "type": "survey"}' for path, open_file in open_forms.items(): XForm.objects.create( xml=open_file.read(), user=self.user, json=self.json, require_auth=False, project=self.project) open_file.close() self._create_water_translated_form()
def test_matching_fields_by_type(self): """ Test get_merged_xform_survey(): should only match when question type matches. """ self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(GROUP_A_MD.replace('group', 'repeat'), self.user, id_string='a') xform2 = self._publish_markdown(GROUP_B_MD, self.user, id_string='b') survey = get_merged_xform_survey([xform1, xform2]) expected = { u'default_language': u'default', u'id_string': u'a', u'children': [{ u'name': u'name', u'label': u'Name', u'type': u'text' }, { u'control': { u'bodyless': True }, u'children': [{ u'name': u'instanceID', u'bind': { u'readonly': u'true()', u'calculate': u"concat('uuid:', uuid())" }, u'type': u'calculate' }], u'name': u'meta', u'type': u'group' }], u'type': u'survey', u'name': u'data', u'sms_keyword': u'a', u'title': u'pyxform_autotesttitle' } # yapf: disable self.assertEqual(survey.to_json_dict(), expected)
def setUp(self): self.user = User.objects.create(username="******") profile, c = UserProfile.objects.get_or_create(user=self.user) profile.require_auth = False profile.save() self.project = get_user_default_project(self.user) absolute_path = get_absolute_path("forms") open_forms = open_all_files(absolute_path) self.json = '{"default_language": "default", ' \ '"id_string": "Water_2011_03_17", "children": [], ' \ '"name": "Water_2011_03_17", ' \ '"title": "Water_2011_03_17", "type": "survey"}' for path, open_file in open_forms.items(): XForm.objects.create(xml=open_file.read(), user=self.user, json=self.json, require_auth=False, project=self.project) open_file.close() self._create_water_translated_form()
def setUp(self): self.user = User.objects.create( username="******", email="*****@*****.**") self.project = UserProfile.objects.create(user=self.user) self.user.set_password("pass") self.project = get_user_default_project(self.user) self.xform1 = DataDictionary() self.xform1.user = self.user self.xform1.project = self.project self.xform1.json = '{"id_string": "yes_or_no", "children": [{"name": '\ '"yesno", "label": "Yes or no?", "type": "text"}],'\ ' "name": "yes_or_no", "title": "yes_or_no", "type'\ '": "survey"}'.strip() self.xform2 = DataDictionary() self.xform2.user = self.user self.xform2.project = self.project self.xform2.json = '{"id_string": "start_time", "children": [{"name":'\ '"start_time", "type": "start"}], "name": "start_t'\ 'ime", "title": "start_time", "type": "survey"}'\ .strip() self._get_xml_for_form(self.xform1) self._get_xml_for_form(self.xform2)
def publish(self, user, id_string=None, created_by=None): """ Publish XLSForm. """ if self.is_valid(): # If a text (csv) representation of the xlsform is present, # this will save the file and pass it instead of the 'xls_file' # field. cleaned_xls_file = None if 'text_xls_form' in self.cleaned_data\ and self.cleaned_data['text_xls_form'].strip(): csv_data = self.cleaned_data['text_xls_form'] # assigning the filename to a random string (quick fix) import random rand_name = "uploaded_form_%s.csv" % ''.join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) cleaned_xls_file = \ default_storage.save( upload_to(None, rand_name, user.username), ContentFile(csv_data.encode())) if 'xls_file' in self.cleaned_data and\ self.cleaned_data['xls_file']: cleaned_xls_file = self.cleaned_data['xls_file'] if 'floip_file' in self.cleaned_data and\ self.cleaned_data['floip_file']: cleaned_xls_file = self.cleaned_data['floip_file'] cleaned_url = ( self.cleaned_data['xls_url'].strip() or self.cleaned_data['dropbox_xls_url'] or self.cleaned_data['csv_url']) if cleaned_url: cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = \ '_'.join(cleaned_xls_file.path.split('/')[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: response = requests.get(cleaned_url) if response.headers.get('content-type') in \ VALID_XLSFORM_CONTENT_TYPES and \ response.status_code < 400: cleaned_xls_file = get_filename(response) cleaned_xls_file = \ upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urlopen(cleaned_url).read()) cleaned_xls_file = \ default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data['project'] if project is None: project = get_user_default_project(user) else: project = self._project cleaned_xml_file = self.cleaned_data['xml_file'] if cleaned_xml_file: return publish_xml_form(cleaned_xml_file, user, project, id_string, created_by or user) if cleaned_xls_file is None: raise forms.ValidationError( _(u"XLSForm not provided, expecting either of these" " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," " 'dropbox_xls_url', 'text_xls_form', 'floip_file'")) # publish the xls return publish_xls_form(cleaned_xls_file, user, project, id_string, created_by or user)
def test_repeat_merged_xform_survey(self): """ Test get_merged_xform_survey() with repeats in xforms. """ self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(GROUP_A_MD.replace('group', 'repeat'), self.user, id_string='a') xform2 = self._publish_markdown(GROUP_B_MD.replace('group', 'repeat'), self.user, id_string='b') survey = get_merged_xform_survey([xform1, xform2]) expected = { u'default_language': u'default', u'id_string': u'a', u'children': [{ u'name': u'name', u'label': u'Name', u'type': u'text' }, { u'children': [{ u'children': [{ u'name': u'female', u'label': u'Female' }, { u'name': u'male', u'label': u'Male' }], u'name': u'gender', u'label': u'Sex', u'type': u'select one' }], u'name': u'info', u'label': u'Info', u'type': u'repeat' }, { u'children': [{ u'children': [{ u'children': [{ u'name': u'female', u'label': u'Female' }, { u'name': u'male', u'label': u'Male' }], u'name': u'gender', u'label': u'Sex', u'type': u'select one' }], u'name': u'person', u'label': u'Person', u'type': u'repeat' }], u'name': u'other', u'label': u'Other', u'type': u'repeat', }, { u'control': { u'bodyless': True }, u'children': [{ u'name': u'instanceID', u'bind': { u'readonly': u'true()', u'calculate': u"concat('uuid:', uuid())" }, u'type': u'calculate' }], u'name': u'meta', u'type': u'group' }], u'type': u'survey', u'name': u'data', u'sms_keyword': u'a', u'title': u'pyxform_autotesttitle' } # yapf: disable self.assertEqual(survey.to_json_dict(), expected)
def __init__(self, xml_file, user): self.xml_file = xml_file self.user = user self.project = get_user_default_project(user)
def test_create_merged_xform(self): """Test creating a merged dataset with the MergedXFormSerializer""" serializer = MergedXFormSerializer(data={}) self.assertFalse(serializer.is_valid(raise_exception=False)) # project is required self.assertEqual(serializer.errors['project'], [u'This field is required.']) # name is required self.assertEqual(serializer.errors['name'], [u'This field is required.']) # At least 2 *different* xforms # 0 xforms self.assertEqual(serializer.errors['xforms'], [u'This field is required.']) self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(MD, self.user, id_string='a') data = { 'xforms': [], 'name': 'Merged Dataset', 'project': "http://testserver.com/api/v1/projects/%s" % self.project.pk, } serializer = MergedXFormSerializer(data=data) self.assertFalse(serializer.is_valid(raise_exception=False)) self.assertNotIn('name', serializer.errors) self.assertNotIn('project', serializer.errors) self.assertEqual(serializer.errors['xforms'], [u'This list may not be empty.']) # 1 xform data['xforms'] = ["http://testserver.com/api/v1/forms/%s" % xform1.pk] serializer = MergedXFormSerializer(data=data) self.assertFalse(serializer.is_valid(raise_exception=False)) self.assertIn(u'This field should have at least two unique xforms.', serializer.errors['xforms']) # same xform twice data['xforms'] = [ "http://testserver.com/api/v1/forms/%s" % xform1.pk, "http://testserver.com/api/v1/forms/%s" % xform1.pk ] serializer = MergedXFormSerializer(data=data) self.assertFalse(serializer.is_valid(raise_exception=False)) self.assertIn(u'This field should have unique xforms', serializer.errors['xforms']) # xform with no matching fields xform3 = self._publish_markdown(A_MD, self.user, id_string='c') data['xforms'] = [ "http://testserver.com/api/v1/forms/%s" % xform1.pk, "http://testserver.com/api/v1/forms/%s" % xform3.pk ] serializer = MergedXFormSerializer(data=data) self.assertFalse(serializer.is_valid(raise_exception=False)) self.assertEqual(serializer.errors['xforms'], [u'No matching fields in xforms.']) # two different xforms xform2 = self._publish_markdown(MD, self.user, id_string='b') data['xforms'] = [ "http://testserver.com/api/v1/forms/%s" % xform1.pk, "http://testserver.com/api/v1/forms/%s" % xform2.pk ] serializer = MergedXFormSerializer(data=data) self.assertTrue(serializer.is_valid(raise_exception=False)) self.assertNotIn('xforms', serializer.errors)
def publish(self, user, id_string=None, created_by=None): """ Publish XLSForm. """ if self.is_valid(): # If a text (csv) representation of the xlsform is present, # this will save the file and pass it instead of the 'xls_file' # field. cleaned_xls_file = None if 'text_xls_form' in self.cleaned_data\ and self.cleaned_data['text_xls_form'].strip(): csv_data = self.cleaned_data['text_xls_form'] # assigning the filename to a random string (quick fix) import random rand_name = "uploaded_form_%s.csv" % ''.join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) cleaned_xls_file = \ default_storage.save( upload_to(None, rand_name, user.username), ContentFile(csv_data.encode())) if 'xls_file' in self.cleaned_data and\ self.cleaned_data['xls_file']: cleaned_xls_file = self.cleaned_data['xls_file'] if 'floip_file' in self.cleaned_data and\ self.cleaned_data['floip_file']: cleaned_xls_file = self.cleaned_data['floip_file'] cleaned_url = (self.cleaned_data['xls_url'].strip() or self.cleaned_data['dropbox_xls_url'] or self.cleaned_data['csv_url']) if cleaned_url: cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = \ '_'.join(cleaned_xls_file.path.split('/')[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: response = requests.get(cleaned_url) if response.headers.get('content-type') in \ VALID_XLSFORM_CONTENT_TYPES and \ response.status_code < 400: cleaned_xls_file = get_filename(response) cleaned_xls_file = \ upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urlopen(cleaned_url).read()) cleaned_xls_file = \ default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data['project'] if project is None: project = get_user_default_project(user) else: project = self._project cleaned_xml_file = self.cleaned_data['xml_file'] if cleaned_xml_file: return publish_xml_form(cleaned_xml_file, user, project, id_string, created_by or user) if cleaned_xls_file is None: raise forms.ValidationError( _(u"XLSForm not provided, expecting either of these" " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," " 'dropbox_xls_url', 'text_xls_form', 'floip_file'")) # publish the xls return publish_xls_form(cleaned_xls_file, user, project, id_string, created_by or user)
def test_get_merged_xform_survey(self): """ Test get_merged_xform_survey() """ with self.assertRaises(serializers.ValidationError): get_merged_xform_survey([]) self.project = get_user_default_project(self.user) xform1 = self._publish_markdown(A_MD, self.user, id_string='a') xform2 = self._publish_markdown(B_MD, self.user, id_string='b') xform3 = self._publish_markdown(MD, self.user, id_string='c') expected = { u'name': u'data', u'title': u'pyxform_autotesttitle', u'sms_keyword': u'a', u'default_language': u'default', u'id_string': u'a', u'type': u'survey', u'children': [{ u'name': u'name', u'label': u'Name', u'type': u'text' }, { u'children': [{ u'name': u'male', u'label': u'Male' }, { u'name': u'female', u'label': u'Female' }], u'name': u'gender', u'label': u'Sex', u'list_name': u'gender', u'type': u'select one' }, { u'control': { u'bodyless': True }, u'children': [{ u'name': u'instanceID', u'bind': { u'readonly': u'true()', u'jr:preload': u"uid" }, u'type': u'calculate' }], u'name': u'meta', u'type': u'group' }] } # yapf: disable with self.assertRaises(serializers.ValidationError): get_merged_xform_survey([xform1]) survey = get_merged_xform_survey([xform1, xform2]) survey_dict = survey.to_json_dict() # this field seems to change 50% of the time expected2 = copy.deepcopy(expected) expected2['children'][1]['children'] = \ [{'name': 'female', 'label': 'Female'}, {'name': 'male', 'label': 'Male'}] self.assertTrue(survey_dict == expected or survey_dict == expected2) # no matching fields with self.assertRaises(serializers.ValidationError): survey = get_merged_xform_survey([xform1, xform3])