def generate_attachments_zip_export(
export_type, extension, username, id_string, export_id=None,
filter_query=None):
xform = XForm.objects.get(user__username=username, id_string=id_string)
attachments = Attachment.objects.filter(instance__xform=xform)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
with NamedTemporaryFile('wb+', prefix='media_zip_export_', suffix='.zip') as temporary_file:
create_attachments_zipfile(attachments, temporary_file=temporary_file)
export_filename = get_storage_class()().save(
file_path,
File(temporary_file, file_path))
dir_name, basename = os.path.split(export_filename)
# get or create export object
if(export_id):
export = Export.objects.get(id=export_id)
else:
export = Export.objects.create(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def generate_attachments_zip_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None):
xform = XForm.objects.get(user__username=username, id_string=id_string)
attachments = Attachment.objects.filter(instance__xform=xform)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
with NamedTemporaryFile('wb+', prefix='media_zip_export_',
suffix='.zip') as temporary_file:
create_attachments_zipfile(attachments, temporary_file=temporary_file)
export_filename = get_storage_class()().save(
file_path, File(temporary_file, file_path))
dir_name, basename = os.path.split(export_filename)
# get or create export object
if (export_id):
export = Export.objects.get(id=export_id)
else:
export = Export.objects.create(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def generate_attachments_zip_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None):
xform = XForm.objects.get(user__username=username, id_string=id_string)
attachments = Attachment.objects.filter(instance__xform=xform)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
export_filename = get_storage_class()().save(file_path, ContentFile(''))
with get_storage_class()().open(export_filename, 'wb') as destination_file:
create_attachments_zipfile(
attachments,
output_file=destination_file,
)
dir_name, basename = os.path.split(export_filename)
# get or create export object
if (export_id):
export = Export.objects.get(id=export_id)
else:
export = Export.objects.create(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def zip_export(request, username, id_string):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if request.GET.get('raw'):
id_string = None
attachments = Attachment.objects.filter(instance__xform=xform)
zip_file = create_attachments_zipfile(attachments)
audit = {"xform": xform.id_string, "export_type": Export.ZIP_EXPORT}
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created ZIP export on '%(id_string)s'.") % {
'id_string': xform.id_string,
}, audit, request)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded ZIP export on '%(id_string)s'.") % {
'id_string': xform.id_string,
}, audit, request)
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name('zip',
id_string,
file_path=zip_file,
use_local_filesystem=True)
return response
def generate_attachments_zip_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None):
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
xform = XForm.objects.get(user__username=username, id_string=id_string)
attachments = Attachment.objects.filter(instance__xform=xform)
zip_file = create_attachments_zipfile(attachments)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
storage = get_storage_class()()
temp_file = open(zip_file)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if (export_id):
export = Export.objects.get(id=export_id)
else:
export = Export.objects.create(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def zip_export(request, username, id_string):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if request.GET.get('raw'):
id_string = None
attachments = Attachment.objects.filter(instance__xform=xform)
zip_file = create_attachments_zipfile(attachments)
audit = {
"xform": xform.id_string,
"export_type": Export.ZIP_EXPORT
}
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created ZIP export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
}, audit, request)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded ZIP export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
}, audit, request)
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name('zip', id_string,
file_path=zip_file,
use_local_filesystem=True)
return response
def test_create_attachments_zipfile_file_too_big(self, rpt_mock):
"""
When a file is larger than what is allowed in settings an exception
should be raised.
"""
self._publish_transportation_form_and_submit_instance()
self.media_file = "1335783522563.jpg"
media_file = os.path.join(
self.this_directory, 'fixtures',
'transportation', 'instances', self.surveys[0], self.media_file)
self.instance = Instance.objects.all()[0]
self.attachment = Attachment.objects.create(
instance=self.instance,
media_file=File(open(media_file, 'rb'), media_file))
create_attachments_zipfile(Attachment.objects.all())
message = (
"Create attachment zip exception", "File is greater than 8 bytes")
self.assertTrue(rpt_mock.called)
rpt_mock.assert_called_with(message[0], message[1])
def test_create_attachments_zipfile_file_too_big(self, rpt_mock):
"""
When a file is larger than what is allowed in settings an exception
should be raised.
"""
self._publish_transportation_form_and_submit_instance()
self.media_file = "1335783522563.jpg"
media_file = os.path.join(self.this_directory, 'fixtures',
'transportation', 'instances',
self.surveys[0], self.media_file)
self.instance = Instance.objects.all()[0]
self.attachment = Attachment.objects.create(instance=self.instance,
media_file=File(
open(media_file, 'rb'),
media_file))
create_attachments_zipfile(Attachment.objects.all())
message = ("Create attachment zip exception",
"File is greater than 8 bytes")
self.assertTrue(rpt_mock.called)
rpt_mock.assert_called_with(message[0], message[1])
def generate_attachments_zip_export(
export_type, extension, username, id_string, export_id=None,
filter_query=None):
xform = XForm.objects.get(user__username=username, id_string=id_string)
attachments = Attachment.objects.filter(instance__xform=xform)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
export_filename = get_storage_class()().save(file_path, ContentFile(''))
with get_storage_class()().open(export_filename, 'wb') as destination_file:
create_attachments_zipfile(
attachments,
output_file=destination_file,
)
dir_name, basename = os.path.split(export_filename)
# get or create export object
if(export_id):
export = Export.objects.get(id=export_id)
else:
export = Export.objects.create(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def generate_attachments_zip_export(
export_type, extension, username, id_string, export_id=None,
filter_query=None):
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
xform = XForm.objects.get(user__username=username, id_string=id_string)
attachments = Attachment.objects.filter(instance__xform=xform)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
storage = get_storage_class()()
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
try:
temp_file = open(zip_file.name)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
finally:
temp_file.close()
finally:
zip_file and zip_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if(export_id):
export = Export.objects.get(id=export_id)
else:
export = Export.objects.create(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def zip_export(request, username, id_string):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if request.GET.get('raw'):
id_string = None
attachments = Attachment.objects.filter(instance__xform=xform)
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
audit = {
"xform": xform.id_string,
"export_type": Export.ZIP_EXPORT
}
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created ZIP export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
}, audit, request)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded ZIP export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
}, audit, request)
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name('zip', id_string)
response.write(FileWrapper(zip_file))
response['Content-Length'] = zip_file.tell()
zip_file.seek(0)
finally:
zip_file and zip_file.close()
return response
def zip_export(request, username, id_string):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
if request.GET.get('raw'):
id_string = None
attachments = Attachment.objects.filter(instance__xform=xform)
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
audit = {
"xform": xform.id_string,
"export_type": Export.ZIP_EXPORT
}
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created ZIP export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
}, audit, request)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded ZIP export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
}, audit, request)
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name('zip', id_string)
response.write(FileWrapper(zip_file))
response['Content-Length'] = zip_file.tell()
zip_file.seek(0)
finally:
zip_file and zip_file.close()
return response
def generate_attachments_zip_export(export_type,
username,
id_string,
export_id=None,
options=None,
xform=None):
"""
Generates zip export of attachments.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
export_type = options.get("extension", export_type)
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
attachments = Attachment.objects.filter(
instance_id__in=[
rec.get('_id')
for rec in dataview.query_data(dataview, all_data=True)
],
instance__deleted_at__isnull=True)
else:
attachments = Attachment.objects.filter(
instance__deleted_at__isnull=True)
if xform.is_merged_dataset:
attachments = attachments.filter(instance__xform_id__in=[
i for i in xform.mergedxform.xforms.filter(
deleted_at__isnull=True).values_list('id', flat=True)
])
else:
attachments = attachments.filter(instance__xform_id=xform.pk)
filename = "%s_%s.%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"),
export_type.lower())
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
try:
temp_file = open(zip_file.name, 'rb')
filename = default_storage.save(file_path,
File(temp_file, file_path))
finally:
temp_file.close()
finally:
if zip_file:
zip_file.close()
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir, export.filename = os.path.split(filename)
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def generate_attachments_zip_export(export_type,
username,
id_string,
export_id=None,
options=None,
xform=None):
"""
Generates zip export of attachments.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
extension = options.get("extension", export_type)
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview, all_data=True)
instances_ids = [rec.get('_id') for rec in records]
attachments = Attachment.objects.filter(instance_id__in=instances_ids)
else:
attachments = Attachment.objects.filter(instance__xform=xform)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
storage = get_storage_class()()
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
try:
temp_file = open(zip_file.name)
export_filename = storage.save(file_path,
File(temp_file, file_path))
finally:
temp_file.close()
finally:
zip_file and zip_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if (export_id):
export = Export.objects.get(id=export_id)
else:
export_options = get_export_options(options)
export = Export.objects.create(xform=xform,
export_type=export_type,
options=export_options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def generate_attachments_zip_export(export_type, username, id_string,
export_id=None, options=None,
xform=None):
"""
Generates zip export of attachments.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
export_type = options.get("extension", export_type)
filter_query = options.get("query")
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
attachments = Attachment.objects.filter(
instance_id__in=[
rec.get('_id')
for rec in dataview.query_data(
dataview, all_data=True, filter_query=filter_query)],
instance__deleted_at__isnull=True)
else:
instance_ids = query_data(xform, fields='["_id"]', query=filter_query)
attachments = Attachment.objects.filter(
instance__deleted_at__isnull=True)
if xform.is_merged_dataset:
attachments = attachments.filter(
instance__xform_id__in=[
i for i in xform.mergedxform.xforms.filter(
deleted_at__isnull=True).values_list(
'id', flat=True)]).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
else:
attachments = attachments.filter(
instance__xform_id=xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
filename = "%s_%s.%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"),
export_type.lower())
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
try:
temp_file = builtins.open(zip_file.name, 'rb')
filename = default_storage.save(
file_path,
File(temp_file, file_path))
finally:
temp_file.close()
finally:
if zip_file:
zip_file.close()
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir, export.filename = os.path.split(filename)
export.internal_status = Export.SUCCESSFUL
export.save()
return export
