def test_attachment_tracking(self): """ Test that when a submission with many attachments is made, we keep track of the total number of attachments expected and if we have received all of them """ md = """ | survey | | | | | | type | name | label | | | image | image1 | Photo | | | image | image2 | Photo | """ self._create_user_and_login() self.xform = self._publish_markdown(md, self.user) xml_string = """ <data id="{}"> <meta> <instanceID>uuid:UJ5jSMAJ1Jz4EszdgHy8n851AsKaqBPO5</instanceID> </meta> <image1>1300221157303.jpg</image1> <image2>1300375832136.jpg</image2> </data> """.format(self.xform.id_string) file_path = "{}/apps/logger/tests/Health_2011_03_13."\ "xml_2011-03-15_20-30-28/1300221157303"\ ".jpg".format(settings.PROJECT_ROOT) media_file = django_file(path=file_path, field_name="image1", content_type="image/jpeg") instance = create_instance(self.user.username, StringIO(xml_string.strip()), media_files=[media_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEquals(instance.json[TOTAL_MEDIA], 2) self.assertEquals(instance.json[MEDIA_COUNT], 1) self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\ "-59/1300375832136.jpg".format(settings.PROJECT_ROOT) media2_file = django_file(path=file2_path, field_name="image2", content_type="image/jpeg") create_instance(self.user.username, StringIO(xml_string.strip()), media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) self.assertEquals(instance2.json[TOTAL_MEDIA], 2) self.assertEquals(instance2.json[MEDIA_COUNT], 2) self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received)
def test_recovers_wrongly_deleted_attachments(self): """ Test that the command recovers the correct attachment """ md = """ | survey | | | | | | type | name | label | | | file | file | File | | | image | image | Image | """ # pylint: disable=invalid-name self._create_user_and_login() xform = self._publish_markdown(md, self.user) xml_string = f""" <data id="{xform.id_string}"> <meta> <instanceID>uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7</instanceID> </meta> <file>Health_2011_03_13.xml_2011-03-15_20-30-28.xml</file> <image>1300221157303.jpg</image> </data> """ media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' '_2011_03_13.xml_2011-03-15_20-30-28/') image_media = django_file( path=f'{media_root}1300221157303.jpg', field_name='image', content_type='image/jpeg') file_media = django_file( path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', field_name='file', content_type='text/xml') instance = create_instance( self.user.username, BytesIO(xml_string.strip().encode('utf-8')), media_files=[file_media, image_media]) self.assertEqual( instance.attachments.filter(deleted_at__isnull=True).count(), 2) attachment = instance.attachments.first() # Soft delete attachment attachment.deleted_at = datetime.now() attachment.deleted_by = self.user attachment.save() self.assertEqual( instance.attachments.filter(deleted_at__isnull=True).count(), 1) # Attempt recovery of attachment recover_deleted_attachments(form_id=instance.xform.id) self.assertEqual( instance.attachments.filter(deleted_at__isnull=True).count(), 2) attachment.refresh_from_db() self.assertIsNone(attachment.deleted_at) self.assertIsNone(attachment.deleted_by)
def test_attachment_tracking_duplicate(self): """ Test that duplicate attachments does not affect if all attachments were received. """ md = """ | survey | | | | | | type | name | label | | | image | image1 | Photo | | | image | image2 | Photo | """ self._create_user_and_login() self.xform = self._publish_markdown(md, self.user) xml_string = """ <data id="{}"> <meta> <instanceID>uuid:UJ5jSMAJ1Jz4EszdgHy8n851AsKaqBPO5</instanceID> </meta> <image1>1300221157303.jpg</image1> <image2>1300375832136.jpg</image2> </data> """.format(self.xform.id_string) file_path = "{}/apps/logger/tests/Health_2011_03_13."\ "xml_2011-03-15_20-30-28/1300221157303"\ ".jpg".format(settings.PROJECT_ROOT) media_file = django_file(path=file_path, field_name="image1", content_type="image/jpeg") instance = create_instance(self.user.username, BytesIO(xml_string.strip().encode('utf-8')), media_files=[media_file]) self.assertFalse(instance.json[MEDIA_ALL_RECEIVED]) self.assertEquals(instance.json[TOTAL_MEDIA], 2) self.assertEquals(instance.json[MEDIA_COUNT], 1) self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) media2_file = django_file(path=file_path, field_name="image1", content_type="image/jpeg") create_instance(self.user.username, BytesIO(xml_string.strip().encode('utf-8')), media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertFalse(instance2.json[MEDIA_ALL_RECEIVED]) self.assertEquals(instance2.json[TOTAL_MEDIA], 2) self.assertEquals(instance2.json[MEDIA_COUNT], 1) self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received)
def restore_backup_from_xml_file(xml_instance_path, username): # check if its a valid xml instance file_name = os.path.basename(xml_instance_path) xml_file = django_file(xml_instance_path, field_name="xml_file", content_type="text/xml") media_files = [] try: date_created = _date_created_from_filename(file_name) except ValueError as e: sys.stderr.write( "Couldn't determine date created from filename: '%s'\n" % file_name) date_created = datetime.now() sys.stdout.write("Creating instance from '%s'\n" % file_name) try: create_instance(username, xml_file, media_files, date_created_override=date_created) return 1 except Exception as e: sys.stderr.write("Could not restore %s, create instance said: %s\n" % (file_name, e)) return 0
def restore_backup_from_xml_file(xml_instance_path, username): # check if its a valid xml instance file_name = os.path.basename(xml_instance_path) xml_file = django_file( xml_instance_path, field_name="xml_file", content_type="text/xml") media_files = [] try: date_created = _date_created_from_filename(file_name) except ValueError: sys.stderr.write( "Couldn't determine date created from filename: '%s'\n" % file_name) date_created = datetime.now() sys.stdout.write("Creating instance from '%s'\n" % file_name) try: create_instance( username, xml_file, media_files, date_created_override=date_created) return 1 except Exception as e: sys.stderr.write( "Could not restore %s, create instance said: %s\n" % (file_name, e)) return 0
def test_attachment_pagination(self): """ Test attachments endpoint pagination support. """ self._submit_transport_instance_w_attachment() self.assertEqual(self.response.status_code, 201) filename = "1335783522564.JPG" path = os.path.join(self.main_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], filename) media_file = django_file(path, 'image2', 'image/jpeg') Attachment.objects.create( instance=self.xform.instances.first(), mimetype='image/jpeg', extension='JPG', name=filename, media_file=media_file) # not using pagination params request = self.factory.get('/', **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # valid page and page_size request = self.factory.get( '/', data={"page": 1, "page_size": 1}, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) # invalid page type request = self.factory.get('/', data={"page": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size type request = self.factory.get('/', data={"page_size": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # invalid page and page_size types request = self.factory.get( '/', data={"page": "invalid", "page_size": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size request = self.factory.get( '/', data={"page": 4, "page_size": 1}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404)
def import_instance_async(username, xform_path, photos, osm_files, status): """ This callback is passed an instance of a XFormInstanceFS. See xform_fs.py for more info. """ with django_file(xform_path, field_name="xml_file", content_type="text/xml") as xml_file: images = [django_file(jpg, field_name="image", content_type="image/jpeg") for jpg in photos] images += [ django_file(osm, field_name='image', content_type='text/xml') for osm in osm_files ] try: create_instance(username, xml_file, images, status) except: pass for i in images: i.close()
def test_direct_image_link_uppercase(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" path = os.path.join(self.main_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], filename) self.attachment.media_file = django_file(path, 'image2', 'image/jpeg') self.attachment.name = filename self.attachment.save() filename = self.attachment.media_file.name file_base, file_extension = os.path.splitext(filename) data = {'filename': file_base + file_extension.upper()} request = self.factory.get('/', data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, basestring)) self.assertEqual(response.data, attachment_url(self.attachment))
def test_date_created_override(self): """ Test that passing a date_created_override when creating and instance will set our date as the date_created """ xml_file_path = os.path.join( settings.ONADATA_DIR, "apps", "logger", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml") xml_file = django_file( xml_file_path, field_name="xml_file", content_type="text/xml") media_files = [] date_created = datetime.strptime("2013-01-01 12:00:00", "%Y-%m-%d %H:%M:%S") instance = create_instance( self.user.username, xml_file, media_files, date_created_override=date_created) self.assertIsNotNone(instance) self.assertEqual(instance.date_created.strftime("%Y-%m-%d %H:%M:%S"), date_created.strftime("%Y-%m-%d %H:%M:%S"))
def test_direct_image_link_uppercase(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" path = os.path.join(self.main_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], filename) self.attachment.media_file = django_file(path, 'image2', 'image/jpeg') self.attachment.name = filename self.attachment.save() filename = self.attachment.media_file.name file_base, file_extension = os.path.splitext(filename) data = { 'filename': file_base + file_extension.upper() } request = self.factory.get('/', data, **self.extra) response = self.retrieve_view(request, pk=self.attachment.pk) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, basestring)) self.assertEqual(response.data, attachment_url(self.attachment))
def restore_backup_from_path(dir_path, username, status): """ Only restores xml submissions, media files are assumed to still be in storage and will be retrieved by the filename stored within the submission """ num_instances = 0 num_restored = 0 for dir_path, dir_names, file_names in os.walk(dir_path): for file_name in file_names: # check if its a valid xml instance xml_instance_path = os.path.join(dir_path, file_name) num_instances += 1 xml_file = django_file( xml_instance_path, field_name="xml_file", content_type="text/xml") media_files = [] date_created = None try: date_created = _date_created_from_filename(file_name) except ValueError as e: sys.stderr.write( "Couldn't determine date created from filename: '%s'\n" % file_name) else: sys.stdout.write("Creating instance from '%s'\n" % file_name) try: create_instance( username, xml_file, media_files, date_created_override=date_created) num_restored += 1 except Exception as e: sys.stderr.write( "Could not restore %s, create instance said: %s\n" % (file_name, e)) return num_instances, num_restored
def test_replaced_attachments_not_tracked(self): """ Test that when a submission with an attachments is made, and later edited, whereby the attachment is replaced, the replaced attachment is no longer tracked for that submission """ md = """ | survey | | | | | | type | name | label | | | file | file | File | | | image | image | Image | """ self._create_user_and_login() self.xform = self._publish_markdown(md, self.user) xml_string = """ <data id="{}"> <meta> <instanceID>uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7</instanceID> </meta> <file>Health_2011_03_13.xml_2011-03-15_20-30-28.xml</file> <image>1300221157303.jpg</image> </data> """.format(self.xform.id_string) media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health' '_2011_03_13.xml_2011-03-15_20-30-28/') image_media = django_file(path=f'{media_root}1300221157303.jpg', field_name='image', content_type='image/jpeg') file_media = django_file( path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml', field_name='file', content_type='text/xml') instance = create_instance(self.user.username, BytesIO(xml_string.strip().encode('utf-8')), media_files=[file_media, image_media]) self.assertTrue(instance.json[MEDIA_ALL_RECEIVED]) self.assertEqual( instance.attachments.filter(deleted_at__isnull=True).count(), 2) self.assertEquals(instance.json[TOTAL_MEDIA], 2) self.assertEquals(instance.json[MEDIA_COUNT], 2) self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media) self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count) self.assertEquals(instance.json[MEDIA_ALL_RECEIVED], instance.media_all_received) patch_value = 'onadata.apps.logger.models.Instance.get_expected_media' with patch(patch_value) as get_expected_media: get_expected_media.return_value = ['1300375832136.jpg'] updated_xml_string = """ <data id="{}"> <meta> <instanceID>uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7</instanceID> </meta> <images> <image1>1300375832136.jpg</image1> </images> </data> """.format(self.xform.id_string) file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011"\ "-03-17_16-29-59/1300375832136.jpg".format( settings.PROJECT_ROOT) media2_file = django_file(path=file2_path, field_name="image1", content_type="image/jpeg") create_instance(self.user.username, BytesIO( updated_xml_string.strip().encode('utf-8')), media_files=[media2_file]) instance2 = Instance.objects.get(pk=instance.pk) self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED]) # Test that only one attachment is recognised for this submission # Since the file is no longer present in the submission self.assertEquals(instance2.json[TOTAL_MEDIA], 1) self.assertEquals(instance2.json[MEDIA_COUNT], 1) self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media) self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count) self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED], instance2.media_all_received)
def test_attachment_pagination(self): """ Test attachments endpoint pagination support. """ self._submit_transport_instance_w_attachment() self.assertEqual(self.response.status_code, 201) filename = "1335783522564.JPG" path = os.path.join(self.main_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], filename) media_file = django_file(path, 'image2', 'image/jpeg') Attachment.objects.create(instance=self.xform.instances.first(), mimetype='image/jpeg', extension='JPG', name=filename, media_file=media_file) # not using pagination params request = self.factory.get('/', **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # valid page and page_size request = self.factory.get('/', data={ "page": 1, "page_size": 1 }, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) # invalid page type request = self.factory.get('/', data={"page": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size type request = self.factory.get('/', data={"page_size": "invalid"}, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) # invalid page and page_size types request = self.factory.get('/', data={ "page": "invalid", "page_size": "invalid" }, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404) # invalid page size request = self.factory.get('/', data={ "page": 4, "page_size": 1 }, **self.extra) response = self.list_view(request) self.assertEqual(response.status_code, 404)
def test_list_view_filter_by_attachment_type(self): self._submit_transport_instance_w_attachment() filename = "1335783522564.JPG" path = os.path.join(self.main_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], filename) media_file = django_file(path, 'video2', 'image/jpeg') Attachment.objects.create(instance=self.xform.instances.first(), mimetype='video/mp4', extension='MP4', name=filename, media_file=media_file) Attachment.objects.create(instance=self.xform.instances.first(), mimetype='application/pdf', extension='PDF', name=filename, media_file=media_file) Attachment.objects.create(instance=self.xform.instances.first(), mimetype='text/plain', extension='TXT', name=filename, media_file=media_file) Attachment.objects.create(instance=self.xform.instances.first(), mimetype='audio/mp3', extension='MP3', name=filename, media_file=media_file) data = {} request = self.factory.get('/', data, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 5) # Apply image Filter data['type'] = 'image' request = self.factory.get('/', data, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]["mimetype"], 'image/jpeg') # Apply audio filter data['type'] = 'audio' request = self.factory.get('/', data, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]["mimetype"], 'audio/mp3') # Apply video filter data['type'] = 'video' request = self.factory.get('/', data, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]["mimetype"], 'video/mp4') # Apply file filter data['type'] = 'document' request = self.factory.get('/', data, **self.extra) response = self.list_view(request) self.assertNotEqual(response.get('Cache-Control'), None) self.assertEqual(response.status_code, 200) self.assertTrue(isinstance(response.data, list)) self.assertEqual(len(response.data), 2) self.assertEqual(response.data[0]["mimetype"], 'application/pdf') self.assertEqual(response.data[1]["mimetype"], 'text/plain')