def test_attachment_tracking(self):
"""
Test that when a submission with many attachments is made,
we keep track of the total number of attachments expected
and if we have received all of them
"""
md = """
| survey | | | |
| | type | name | label |
| | image | image1 | Photo |
| | image | image2 | Photo |
"""
self._create_user_and_login()
self.xform = self._publish_markdown(md, self.user)
xml_string = """
<data id="{}">
<meta>
<instanceID>uuid:UJ5jSMAJ1Jz4EszdgHy8n851AsKaqBPO5</instanceID>
</meta>
<image1>1300221157303.jpg</image1>
<image2>1300375832136.jpg</image2>
</data>
""".format(self.xform.id_string)
file_path = "{}/apps/logger/tests/Health_2011_03_13."\
"xml_2011-03-15_20-30-28/1300221157303"\
".jpg".format(settings.PROJECT_ROOT)
media_file = django_file(path=file_path,
field_name="image1",
content_type="image/jpeg")
instance = create_instance(self.user.username,
StringIO(xml_string.strip()),
media_files=[media_file])
self.assertFalse(instance.json[MEDIA_ALL_RECEIVED])
self.assertEquals(instance.json[TOTAL_MEDIA], 2)
self.assertEquals(instance.json[MEDIA_COUNT], 1)
self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media)
self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count)
self.assertEquals(instance.json[MEDIA_ALL_RECEIVED],
instance.media_all_received)
file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\
"-59/1300375832136.jpg".format(settings.PROJECT_ROOT)
media2_file = django_file(path=file2_path,
field_name="image2",
content_type="image/jpeg")
create_instance(self.user.username,
StringIO(xml_string.strip()),
media_files=[media2_file])
instance2 = Instance.objects.get(pk=instance.pk)
self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED])
self.assertEquals(instance2.json[TOTAL_MEDIA], 2)
self.assertEquals(instance2.json[MEDIA_COUNT], 2)
self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media)
self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count)
self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED],
instance2.media_all_received)
def test_recovers_wrongly_deleted_attachments(self):
"""
Test that the command recovers the correct
attachment
"""
md = """
| survey | | | |
| | type | name | label |
| | file | file | File |
| | image | image | Image |
""" # pylint: disable=invalid-name
self._create_user_and_login()
xform = self._publish_markdown(md, self.user)
xml_string = f"""
<data id="{xform.id_string}">
<meta>
<instanceID>uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7</instanceID>
</meta>
<file>Health_2011_03_13.xml_2011-03-15_20-30-28.xml</file>
<image>1300221157303.jpg</image>
</data>
"""
media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health'
'_2011_03_13.xml_2011-03-15_20-30-28/')
image_media = django_file(
path=f'{media_root}1300221157303.jpg', field_name='image',
content_type='image/jpeg')
file_media = django_file(
path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml',
field_name='file', content_type='text/xml')
instance = create_instance(
self.user.username,
BytesIO(xml_string.strip().encode('utf-8')),
media_files=[file_media, image_media])
self.assertEqual(
instance.attachments.filter(deleted_at__isnull=True).count(), 2)
attachment = instance.attachments.first()
# Soft delete attachment
attachment.deleted_at = datetime.now()
attachment.deleted_by = self.user
attachment.save()
self.assertEqual(
instance.attachments.filter(deleted_at__isnull=True).count(), 1)
# Attempt recovery of attachment
recover_deleted_attachments(form_id=instance.xform.id)
self.assertEqual(
instance.attachments.filter(deleted_at__isnull=True).count(), 2)
attachment.refresh_from_db()
self.assertIsNone(attachment.deleted_at)
self.assertIsNone(attachment.deleted_by)
def test_attachment_tracking_duplicate(self):
"""
Test that duplicate attachments does not affect if all attachments were
received.
"""
md = """
| survey | | | |
| | type | name | label |
| | image | image1 | Photo |
| | image | image2 | Photo |
"""
self._create_user_and_login()
self.xform = self._publish_markdown(md, self.user)
xml_string = """
<data id="{}">
<meta>
<instanceID>uuid:UJ5jSMAJ1Jz4EszdgHy8n851AsKaqBPO5</instanceID>
</meta>
<image1>1300221157303.jpg</image1>
<image2>1300375832136.jpg</image2>
</data>
""".format(self.xform.id_string)
file_path = "{}/apps/logger/tests/Health_2011_03_13."\
"xml_2011-03-15_20-30-28/1300221157303"\
".jpg".format(settings.PROJECT_ROOT)
media_file = django_file(path=file_path,
field_name="image1",
content_type="image/jpeg")
instance = create_instance(self.user.username,
BytesIO(xml_string.strip().encode('utf-8')),
media_files=[media_file])
self.assertFalse(instance.json[MEDIA_ALL_RECEIVED])
self.assertEquals(instance.json[TOTAL_MEDIA], 2)
self.assertEquals(instance.json[MEDIA_COUNT], 1)
self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media)
self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count)
self.assertEquals(instance.json[MEDIA_ALL_RECEIVED],
instance.media_all_received)
media2_file = django_file(path=file_path,
field_name="image1",
content_type="image/jpeg")
create_instance(self.user.username,
BytesIO(xml_string.strip().encode('utf-8')),
media_files=[media2_file])
instance2 = Instance.objects.get(pk=instance.pk)
self.assertFalse(instance2.json[MEDIA_ALL_RECEIVED])
self.assertEquals(instance2.json[TOTAL_MEDIA], 2)
self.assertEquals(instance2.json[MEDIA_COUNT], 1)
self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media)
self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count)
self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED],
instance2.media_all_received)
def restore_backup_from_xml_file(xml_instance_path, username):
# check if its a valid xml instance
file_name = os.path.basename(xml_instance_path)
xml_file = django_file(xml_instance_path,
field_name="xml_file",
content_type="text/xml")
media_files = []
try:
date_created = _date_created_from_filename(file_name)
except ValueError as e:
sys.stderr.write(
"Couldn't determine date created from filename: '%s'\n" %
file_name)
date_created = datetime.now()
sys.stdout.write("Creating instance from '%s'\n" % file_name)
try:
create_instance(username,
xml_file,
media_files,
date_created_override=date_created)
return 1
except Exception as e:
sys.stderr.write("Could not restore %s, create instance said: %s\n" %
(file_name, e))
return 0
def restore_backup_from_xml_file(xml_instance_path, username):
# check if its a valid xml instance
file_name = os.path.basename(xml_instance_path)
xml_file = django_file(
xml_instance_path,
field_name="xml_file",
content_type="text/xml")
media_files = []
try:
date_created = _date_created_from_filename(file_name)
except ValueError:
sys.stderr.write(
"Couldn't determine date created from filename: '%s'\n" %
file_name)
date_created = datetime.now()
sys.stdout.write("Creating instance from '%s'\n" % file_name)
try:
create_instance(
username, xml_file, media_files,
date_created_override=date_created)
return 1
except Exception as e:
sys.stderr.write(
"Could not restore %s, create instance said: %s\n" %
(file_name, e))
return 0
def test_attachment_pagination(self):
"""
Test attachments endpoint pagination support.
"""
self._submit_transport_instance_w_attachment()
self.assertEqual(self.response.status_code, 201)
filename = "1335783522564.JPG"
path = os.path.join(self.main_directory, 'fixtures', 'transportation',
'instances', self.surveys[0], filename)
media_file = django_file(path, 'image2', 'image/jpeg')
Attachment.objects.create(
instance=self.xform.instances.first(),
mimetype='image/jpeg',
extension='JPG',
name=filename,
media_file=media_file)
# not using pagination params
request = self.factory.get('/', **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 2)
# valid page and page_size
request = self.factory.get(
'/', data={"page": 1, "page_size": 1}, **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 1)
# invalid page type
request = self.factory.get('/', data={"page": "invalid"}, **self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 404)
# invalid page size type
request = self.factory.get('/', data={"page_size": "invalid"},
**self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 2)
# invalid page and page_size types
request = self.factory.get(
'/', data={"page": "invalid", "page_size": "invalid"},
**self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 404)
# invalid page size
request = self.factory.get(
'/', data={"page": 4, "page_size": 1}, **self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 404)
def import_instance_async(username, xform_path, photos, osm_files, status):
"""
This callback is passed an instance of a XFormInstanceFS.
See xform_fs.py for more info.
"""
with django_file(xform_path, field_name="xml_file",
content_type="text/xml") as xml_file:
images = [django_file(jpg, field_name="image",
content_type="image/jpeg") for jpg in photos]
images += [
django_file(osm, field_name='image',
content_type='text/xml')
for osm in osm_files
]
try:
create_instance(username, xml_file, images, status)
except:
pass
for i in images:
i.close()
def test_direct_image_link_uppercase(self):
self._submit_transport_instance_w_attachment()
filename = "1335783522564.JPG"
path = os.path.join(self.main_directory, 'fixtures', 'transportation',
'instances', self.surveys[0], filename)
self.attachment.media_file = django_file(path, 'image2', 'image/jpeg')
self.attachment.name = filename
self.attachment.save()
filename = self.attachment.media_file.name
file_base, file_extension = os.path.splitext(filename)
data = {'filename': file_base + file_extension.upper()}
request = self.factory.get('/', data, **self.extra)
response = self.retrieve_view(request, pk=self.attachment.pk)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, basestring))
self.assertEqual(response.data, attachment_url(self.attachment))
def test_date_created_override(self):
"""
Test that passing a date_created_override when creating and instance
will set our date as the date_created
"""
xml_file_path = os.path.join(
settings.ONADATA_DIR, "apps", "logger", "fixtures",
"tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml")
xml_file = django_file(
xml_file_path, field_name="xml_file", content_type="text/xml")
media_files = []
date_created = datetime.strptime("2013-01-01 12:00:00",
"%Y-%m-%d %H:%M:%S")
instance = create_instance(
self.user.username, xml_file, media_files,
date_created_override=date_created)
self.assertIsNotNone(instance)
self.assertEqual(instance.date_created.strftime("%Y-%m-%d %H:%M:%S"),
date_created.strftime("%Y-%m-%d %H:%M:%S"))
def test_date_created_override(self):
"""
Test that passing a date_created_override when creating and instance
will set our date as the date_created
"""
xml_file_path = os.path.join(
settings.ONADATA_DIR, "apps", "logger", "fixtures",
"tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml")
xml_file = django_file(
xml_file_path, field_name="xml_file", content_type="text/xml")
media_files = []
date_created = datetime.strptime("2013-01-01 12:00:00",
"%Y-%m-%d %H:%M:%S")
instance = create_instance(
self.user.username, xml_file, media_files,
date_created_override=date_created)
self.assertIsNotNone(instance)
self.assertEqual(instance.date_created.strftime("%Y-%m-%d %H:%M:%S"),
date_created.strftime("%Y-%m-%d %H:%M:%S"))
def test_direct_image_link_uppercase(self):
self._submit_transport_instance_w_attachment()
filename = "1335783522564.JPG"
path = os.path.join(self.main_directory, 'fixtures', 'transportation',
'instances', self.surveys[0], filename)
self.attachment.media_file = django_file(path, 'image2', 'image/jpeg')
self.attachment.name = filename
self.attachment.save()
filename = self.attachment.media_file.name
file_base, file_extension = os.path.splitext(filename)
data = {
'filename': file_base + file_extension.upper()
}
request = self.factory.get('/', data, **self.extra)
response = self.retrieve_view(request, pk=self.attachment.pk)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, basestring))
self.assertEqual(response.data, attachment_url(self.attachment))
def restore_backup_from_path(dir_path, username, status):
"""
Only restores xml submissions, media files are assumed to still be in
storage and will be retrieved by the filename stored within the submission
"""
num_instances = 0
num_restored = 0
for dir_path, dir_names, file_names in os.walk(dir_path):
for file_name in file_names:
# check if its a valid xml instance
xml_instance_path = os.path.join(dir_path, file_name)
num_instances += 1
xml_file = django_file(
xml_instance_path,
field_name="xml_file",
content_type="text/xml")
media_files = []
date_created = None
try:
date_created = _date_created_from_filename(file_name)
except ValueError as e:
sys.stderr.write(
"Couldn't determine date created from filename: '%s'\n" %
file_name)
else:
sys.stdout.write("Creating instance from '%s'\n" % file_name)
try:
create_instance(
username, xml_file, media_files,
date_created_override=date_created)
num_restored += 1
except Exception as e:
sys.stderr.write(
"Could not restore %s, create instance said: %s\n" %
(file_name, e))
return num_instances, num_restored
def test_replaced_attachments_not_tracked(self):
"""
Test that when a submission with an attachments is made,
and later edited, whereby the attachment is replaced,
the replaced attachment is no longer tracked for that
submission
"""
md = """
| survey | | | |
| | type | name | label |
| | file | file | File |
| | image | image | Image |
"""
self._create_user_and_login()
self.xform = self._publish_markdown(md, self.user)
xml_string = """
<data id="{}">
<meta>
<instanceID>uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7</instanceID>
</meta>
<file>Health_2011_03_13.xml_2011-03-15_20-30-28.xml</file>
<image>1300221157303.jpg</image>
</data>
""".format(self.xform.id_string)
media_root = (f'{settings.PROJECT_ROOT}/apps/logger/tests/Health'
'_2011_03_13.xml_2011-03-15_20-30-28/')
image_media = django_file(path=f'{media_root}1300221157303.jpg',
field_name='image',
content_type='image/jpeg')
file_media = django_file(
path=f'{media_root}Health_2011_03_13.xml_2011-03-15_20-30-28.xml',
field_name='file',
content_type='text/xml')
instance = create_instance(self.user.username,
BytesIO(xml_string.strip().encode('utf-8')),
media_files=[file_media, image_media])
self.assertTrue(instance.json[MEDIA_ALL_RECEIVED])
self.assertEqual(
instance.attachments.filter(deleted_at__isnull=True).count(), 2)
self.assertEquals(instance.json[TOTAL_MEDIA], 2)
self.assertEquals(instance.json[MEDIA_COUNT], 2)
self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media)
self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count)
self.assertEquals(instance.json[MEDIA_ALL_RECEIVED],
instance.media_all_received)
patch_value = 'onadata.apps.logger.models.Instance.get_expected_media'
with patch(patch_value) as get_expected_media:
get_expected_media.return_value = ['1300375832136.jpg']
updated_xml_string = """
<data id="{}">
<meta>
<instanceID>uuid:UJ5jz4EszdgH8uhy8nss1AsKaqBPO5VN7</instanceID>
</meta>
<images>
<image1>1300375832136.jpg</image1>
</images>
</data>
""".format(self.xform.id_string)
file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011"\
"-03-17_16-29-59/1300375832136.jpg".format(
settings.PROJECT_ROOT)
media2_file = django_file(path=file2_path,
field_name="image1",
content_type="image/jpeg")
create_instance(self.user.username,
BytesIO(
updated_xml_string.strip().encode('utf-8')),
media_files=[media2_file])
instance2 = Instance.objects.get(pk=instance.pk)
self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED])
# Test that only one attachment is recognised for this submission
# Since the file is no longer present in the submission
self.assertEquals(instance2.json[TOTAL_MEDIA], 1)
self.assertEquals(instance2.json[MEDIA_COUNT], 1)
self.assertEquals(instance2.json[TOTAL_MEDIA],
instance2.total_media)
self.assertEquals(instance2.json[MEDIA_COUNT],
instance2.media_count)
self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED],
instance2.media_all_received)
def test_attachment_pagination(self):
"""
Test attachments endpoint pagination support.
"""
self._submit_transport_instance_w_attachment()
self.assertEqual(self.response.status_code, 201)
filename = "1335783522564.JPG"
path = os.path.join(self.main_directory, 'fixtures', 'transportation',
'instances', self.surveys[0], filename)
media_file = django_file(path, 'image2', 'image/jpeg')
Attachment.objects.create(instance=self.xform.instances.first(),
mimetype='image/jpeg',
extension='JPG',
name=filename,
media_file=media_file)
# not using pagination params
request = self.factory.get('/', **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 2)
# valid page and page_size
request = self.factory.get('/',
data={
"page": 1,
"page_size": 1
},
**self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 1)
# invalid page type
request = self.factory.get('/', data={"page": "invalid"}, **self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 404)
# invalid page size type
request = self.factory.get('/',
data={"page_size": "invalid"},
**self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 2)
# invalid page and page_size types
request = self.factory.get('/',
data={
"page": "invalid",
"page_size": "invalid"
},
**self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 404)
# invalid page size
request = self.factory.get('/',
data={
"page": 4,
"page_size": 1
},
**self.extra)
response = self.list_view(request)
self.assertEqual(response.status_code, 404)
def test_list_view_filter_by_attachment_type(self):
self._submit_transport_instance_w_attachment()
filename = "1335783522564.JPG"
path = os.path.join(self.main_directory, 'fixtures', 'transportation',
'instances', self.surveys[0], filename)
media_file = django_file(path, 'video2', 'image/jpeg')
Attachment.objects.create(instance=self.xform.instances.first(),
mimetype='video/mp4',
extension='MP4',
name=filename,
media_file=media_file)
Attachment.objects.create(instance=self.xform.instances.first(),
mimetype='application/pdf',
extension='PDF',
name=filename,
media_file=media_file)
Attachment.objects.create(instance=self.xform.instances.first(),
mimetype='text/plain',
extension='TXT',
name=filename,
media_file=media_file)
Attachment.objects.create(instance=self.xform.instances.first(),
mimetype='audio/mp3',
extension='MP3',
name=filename,
media_file=media_file)
data = {}
request = self.factory.get('/', data, **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 5)
# Apply image Filter
data['type'] = 'image'
request = self.factory.get('/', data, **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]["mimetype"], 'image/jpeg')
# Apply audio filter
data['type'] = 'audio'
request = self.factory.get('/', data, **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]["mimetype"], 'audio/mp3')
# Apply video filter
data['type'] = 'video'
request = self.factory.get('/', data, **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 1)
self.assertEqual(response.data[0]["mimetype"], 'video/mp4')
# Apply file filter
data['type'] = 'document'
request = self.factory.get('/', data, **self.extra)
response = self.list_view(request)
self.assertNotEqual(response.get('Cache-Control'), None)
self.assertEqual(response.status_code, 200)
self.assertTrue(isinstance(response.data, list))
self.assertEqual(len(response.data), 2)
self.assertEqual(response.data[0]["mimetype"], 'application/pdf')
self.assertEqual(response.data[1]["mimetype"], 'text/plain')