def generate_attachments_zip_export( export_type, extension, username, id_string, export_id=None, filter_query=None): xform = XForm.objects.get(user__username=username, id_string=id_string) attachments = Attachment.objects.filter(instance__xform=xform) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join( username, 'exports', id_string, export_type, filename) with NamedTemporaryFile('wb+', prefix='media_zip_export_', suffix='.zip') as temporary_file: create_attachments_zipfile(attachments, temporary_file=temporary_file) export_filename = get_storage_class()().save( file_path, File(temporary_file, file_path)) dir_name, basename = os.path.split(export_filename) # get or create export object if(export_id): export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def generate_attachments_zip_export(export_type, extension, username, id_string, export_id=None, filter_query=None): xform = XForm.objects.get(user__username=username, id_string=id_string) attachments = Attachment.objects.filter(instance__xform=xform) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join(username, 'exports', id_string, export_type, filename) with NamedTemporaryFile('wb+', prefix='media_zip_export_', suffix='.zip') as temporary_file: create_attachments_zipfile(attachments, temporary_file=temporary_file) export_filename = get_storage_class()().save( file_path, File(temporary_file, file_path)) dir_name, basename = os.path.split(export_filename) # get or create export object if (export_id): export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def generate_attachments_zip_export(export_type, extension, username, id_string, export_id=None, filter_query=None): xform = XForm.objects.get(user__username=username, id_string=id_string) attachments = Attachment.objects.filter(instance__xform=xform) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join(username, 'exports', id_string, export_type, filename) export_filename = get_storage_class()().save(file_path, ContentFile('')) with get_storage_class()().open(export_filename, 'wb') as destination_file: create_attachments_zipfile( attachments, output_file=destination_file, ) dir_name, basename = os.path.split(export_filename) # get or create export object if (export_id): export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def zip_export(request, username, id_string): owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if request.GET.get('raw'): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) zip_file = create_attachments_zipfile(attachments) audit = {"xform": xform.id_string, "export_type": Export.ZIP_EXPORT} audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name('zip', id_string, file_path=zip_file, use_local_filesystem=True) return response
def generate_attachments_zip_export(export_type, extension, username, id_string, export_id=None, filter_query=None): # TODO resolve circular import from onadata.apps.viewer.models.export import Export xform = XForm.objects.get(user__username=username, id_string=id_string) attachments = Attachment.objects.filter(instance__xform=xform) zip_file = create_attachments_zipfile(attachments) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join(username, 'exports', id_string, export_type, filename) storage = get_storage_class()() temp_file = open(zip_file) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if (export_id): export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def zip_export(request, username, id_string): owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if request.GET.get('raw'): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) zip_file = create_attachments_zipfile(attachments) audit = { "xform": xform.id_string, "export_type": Export.ZIP_EXPORT } audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name('zip', id_string, file_path=zip_file, use_local_filesystem=True) return response
def test_create_attachments_zipfile_file_too_big(self, rpt_mock): """ When a file is larger than what is allowed in settings an exception should be raised. """ self._publish_transportation_form_and_submit_instance() self.media_file = "1335783522563.jpg" media_file = os.path.join( self.this_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], self.media_file) self.instance = Instance.objects.all()[0] self.attachment = Attachment.objects.create( instance=self.instance, media_file=File(open(media_file, 'rb'), media_file)) create_attachments_zipfile(Attachment.objects.all()) message = ( "Create attachment zip exception", "File is greater than 8 bytes") self.assertTrue(rpt_mock.called) rpt_mock.assert_called_with(message[0], message[1])
def test_create_attachments_zipfile_file_too_big(self, rpt_mock): """ When a file is larger than what is allowed in settings an exception should be raised. """ self._publish_transportation_form_and_submit_instance() self.media_file = "1335783522563.jpg" media_file = os.path.join(self.this_directory, 'fixtures', 'transportation', 'instances', self.surveys[0], self.media_file) self.instance = Instance.objects.all()[0] self.attachment = Attachment.objects.create(instance=self.instance, media_file=File( open(media_file, 'rb'), media_file)) create_attachments_zipfile(Attachment.objects.all()) message = ("Create attachment zip exception", "File is greater than 8 bytes") self.assertTrue(rpt_mock.called) rpt_mock.assert_called_with(message[0], message[1])
def generate_attachments_zip_export( export_type, extension, username, id_string, export_id=None, filter_query=None): xform = XForm.objects.get(user__username=username, id_string=id_string) attachments = Attachment.objects.filter(instance__xform=xform) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join( username, 'exports', id_string, export_type, filename) export_filename = get_storage_class()().save(file_path, ContentFile('')) with get_storage_class()().open(export_filename, 'wb') as destination_file: create_attachments_zipfile( attachments, output_file=destination_file, ) dir_name, basename = os.path.split(export_filename) # get or create export object if(export_id): export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def generate_attachments_zip_export( export_type, extension, username, id_string, export_id=None, filter_query=None): # TODO resolve circular import from onadata.apps.viewer.models.export import Export xform = XForm.objects.get(user__username=username, id_string=id_string) attachments = Attachment.objects.filter(instance__xform=xform) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join( username, 'exports', id_string, export_type, filename) storage = get_storage_class()() zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: temp_file = open(zip_file.name) export_filename = storage.save( file_path, File(temp_file, file_path)) finally: temp_file.close() finally: zip_file and zip_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if(export_id): export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def zip_export(request, username, id_string): owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if request.GET.get('raw'): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) zip_file = None try: zip_file = create_attachments_zipfile(attachments) audit = { "xform": xform.id_string, "export_type": Export.ZIP_EXPORT } audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name('zip', id_string) response.write(FileWrapper(zip_file)) response['Content-Length'] = zip_file.tell() zip_file.seek(0) finally: zip_file and zip_file.close() return response
def zip_export(request, username, id_string): owner = get_object_or_404(User, username=username) xform = get_object_or_404(XForm, id_string=id_string, user=owner) helper_auth_helper(request) if not has_permission(xform, owner, request): return HttpResponseForbidden(_(u'Not shared.')) if request.GET.get('raw'): id_string = None attachments = Attachment.objects.filter(instance__xform=xform) zip_file = None try: zip_file = create_attachments_zipfile(attachments) audit = { "xform": xform.id_string, "export_type": Export.ZIP_EXPORT } audit_log( Actions.EXPORT_CREATED, request.user, owner, _("Created ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) # log download as well audit_log( Actions.EXPORT_DOWNLOADED, request.user, owner, _("Downloaded ZIP export on '%(id_string)s'.") % { 'id_string': xform.id_string, }, audit, request) if request.GET.get('raw'): id_string = None response = response_with_mimetype_and_name('zip', id_string) response.write(FileWrapper(zip_file)) response['Content-Length'] = zip_file.tell() zip_file.seek(0) finally: zip_file and zip_file.close() return response
def generate_attachments_zip_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates zip export of attachments. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. ext: File extension of the generated export """ export_type = options.get("extension", export_type) if xform is None: xform = XForm.objects.get(user__username=username, id_string=id_string) if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ rec.get('_id') for rec in dataview.query_data(dataview, all_data=True) ], instance__deleted_at__isnull=True) else: attachments = Attachment.objects.filter( instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter(instance__xform_id__in=[ i for i in xform.mergedxform.xforms.filter( deleted_at__isnull=True).values_list('id', flat=True) ]) else: attachments = attachments.filter(instance__xform_id=xform.pk) filename = "%s_%s.%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), export_type.lower()) file_path = os.path.join(username, 'exports', id_string, export_type, filename) zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: temp_file = open(zip_file.name, 'rb') filename = default_storage.save(file_path, File(temp_file, file_path)) finally: temp_file.close() finally: if zip_file: zip_file.close() export = get_or_create_export(export_id, xform, export_type, options) export.filedir, export.filename = os.path.split(filename) export.internal_status = Export.SUCCESSFUL export.save() return export
def generate_attachments_zip_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates zip export of attachments. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. ext: File extension of the generated export """ extension = options.get("extension", export_type) if xform is None: xform = XForm.objects.get(user__username=username, id_string=id_string) if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True) instances_ids = [rec.get('_id') for rec in records] attachments = Attachment.objects.filter(instance_id__in=instances_ids) else: attachments = Attachment.objects.filter(instance__xform=xform) basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension file_path = os.path.join(username, 'exports', id_string, export_type, filename) storage = get_storage_class()() zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: temp_file = open(zip_file.name) export_filename = storage.save(file_path, File(temp_file, file_path)) finally: temp_file.close() finally: zip_file and zip_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if (export_id): export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) export = Export.objects.create(xform=xform, export_type=export_type, options=export_options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL export.save() return export
def generate_attachments_zip_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates zip export of attachments. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. ext: File extension of the generated export """ export_type = options.get("extension", export_type) filter_query = options.get("query") if xform is None: xform = XForm.objects.get(user__username=username, id_string=id_string) if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ rec.get('_id') for rec in dataview.query_data( dataview, all_data=True, filter_query=filter_query)], instance__deleted_at__isnull=True) else: instance_ids = query_data(xform, fields='["_id"]', query=filter_query) attachments = Attachment.objects.filter( instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter( instance__xform_id__in=[ i for i in xform.mergedxform.xforms.filter( deleted_at__isnull=True).values_list( 'id', flat=True)]).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) else: attachments = attachments.filter( instance__xform_id=xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) filename = "%s_%s.%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), export_type.lower()) file_path = os.path.join( username, 'exports', id_string, export_type, filename) zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: temp_file = builtins.open(zip_file.name, 'rb') filename = default_storage.save( file_path, File(temp_file, file_path)) finally: temp_file.close() finally: if zip_file: zip_file.close() export = get_or_create_export(export_id, xform, export_type, options) export.filedir, export.filename = os.path.split(filename) export.internal_status = Export.SUCCESSFUL export.save() return export