def test_generate_attachments_zip_export(self):
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(os.path.dirname(api_tests.__file__), 'fixtures',
'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {"extension": Export.ZIP_EXPORT}
export = generate_attachments_zip_export(Export.ZIP_EXPORT,
self.user.username,
self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
# deleted submission
submission = self.xform.instances.filter().first()
submission.deleted_at = timezone.now()
submission.save()
export = generate_attachments_zip_export(Export.ZIP_EXPORT,
self.user.username,
self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertFalse(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
def test_generate_attachments_zip_export(self):
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(
os.path.dirname(api_tests.__file__), 'fixtures', 'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {"extension": Export.ZIP_EXPORT}
export = generate_attachments_zip_export(
Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
# deleted submission
submission = self.xform.instances.filter().first()
submission.deleted_at = timezone.now()
submission.save()
export = generate_attachments_zip_export(
Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertFalse(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
def create_zip_export(username, id_string, export_id, **options):
"""
Attachments zip export task.
"""
export = _get_export_object(export_id)
try:
gen_export = generate_attachments_zip_export(
Export.ZIP_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"Zip Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e)
raise
else:
if not settings.TESTING_MODE:
delete_export.apply_async(
(), {'export_id': gen_export.id},
countdown=settings.ZIP_EXPORT_COUNTDOWN)
return gen_export.id
def create_zip_export(username, id_string, export_id, **options):
"""
Attachments zip export task.
"""
export = _get_export_object(export_id)
try:
gen_export = generate_attachments_zip_export(
Export.ZIP_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"Zip Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e)
raise
else:
if not settings.TESTING_MODE:
delete_export.apply_async(
(), {'export_id': gen_export.id},
countdown=settings.ZIP_EXPORT_COUNTDOWN)
return gen_export.id
def create_zip_export(username, id_string, export_id, query=None):
export = Export.objects.get(id=export_id)
try:
gen_export = generate_attachments_zip_export(
Export.ZIP_EXPORT, 'zip', username, id_string, export_id, query)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("Zip Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e)
raise
else:
if not settings.TESTING_MODE:
delete_export.apply_async(
(), {'export_id': gen_export.id},
countdown=settings.ZIP_EXPORT_COUNTDOWN)
return gen_export.id
def _generate_new_export(request,
xform,
query,
export_type,
dataview_pk=False):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
options = {
"extension": extension,
"username": xform.user.username,
"id_string": xform.id_string,
}
if query:
options['query'] = query
options["dataview_pk"] = dataview_pk
if export_type == Export.GOOGLE_SHEETS_EXPORT:
options['google_credentials'] = \
_get_google_credential(request).to_json()
try:
if export_type == Export.EXTERNAL_EXPORT:
options['token'] = request.GET.get('token')
options['data_id'] = request.GET.get('data_id')
options['meta'] = request.GET.get('meta')
export = generate_external_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.ZIP_EXPORT:
export = generate_attachments_zip_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
else:
options.update(parse_request_export_options(request.query_params))
export = generate_export(export_type, xform, None, options)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return async_status(FAILED, str(e))
except SPSSIOError as e:
raise exceptions.ParseError(str(e))
else:
return export
def test_generate_filtered_attachments_zip_export(self):
"""Test media zip file export filters attachments"""
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(os.path.dirname(api_tests.__file__), 'fixtures',
'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {
"extension": Export.ZIP_EXPORT,
"query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'
}
filter_query = options.get("query")
instance_ids = query_data(self.xform,
fields='["_id"]',
query=filter_query)
export = generate_attachments_zip_export(Export.ZIP_EXPORT,
self.user.username,
self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
filtered_attachments = Attachment.objects.filter(
instance__xform_id=self.xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
self.assertNotEqual(Attachment.objects.count(),
filtered_attachments.count())
for a in filtered_attachments:
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
# export with no query
options.pop('query')
export1 = generate_attachments_zip_export(Export.ZIP_EXPORT,
self.user.username,
self.xform.id_string, None,
options)
self.assertTrue(export1.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
def test_generate_filtered_attachments_zip_export(self):
"""Test media zip file export filters attachments"""
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(
os.path.dirname(api_tests.__file__), 'fixtures', 'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {
"extension": Export.ZIP_EXPORT,
"query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'}
filter_query = options.get("query")
instance_ids = query_data(
self.xform, fields='["_id"]', query=filter_query)
export = generate_attachments_zip_export(
Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
filtered_attachments = Attachment.objects.filter(
instance__xform_id=self.xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
self.assertNotEqual(
Attachment.objects.count(), filtered_attachments.count())
for a in filtered_attachments:
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
# export with no query
options.pop('query')
export1 = generate_attachments_zip_export(
Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None,
options)
self.assertTrue(export1.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
