class TemporaryFileUploadHandler(FileUploadHandler): """ Upload handler that streams data into a temporary file. """ def new_file(self, *args, **kwargs): """ Create the file object to append to as data is coming in. """ super().new_file(*args, **kwargs) self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset, self.content_type_extra) def receive_data_chunk(self, raw_data, start): self.file.write(raw_data) def file_complete(self, file_size): self.file.seek(0) self.file.size = file_size return self.file def upload_interrupted(self): if hasattr(self, 'file'): temp_location = self.file.temporary_file_path() try: self.file.close() os.remove(temp_location) except FileNotFoundError: pass
class FileWriteTest(TestCase): def setUp(self): self.text = "Spam Spam Spam Spam.\n" self.digest = hashlib.sha1(self.text).hexdigest() self.tempfile = TemporaryUploadedFile('spam4.txt', 'text/plain', len(self.text), 'utf-8') self.tempfile.file.write(self.text) self.tempfile.file.seek(0) def test(self): new_upload = Upload(file=self.tempfile) new_upload.save() # Upload has been saved to the database. self.assert_(new_upload.pk) # Upload contains correct content. self.assertEqual(new_upload.file.read(), self.text) # Filename is the hash of the file contents. self.assert_(new_upload.file.name.startswith(self.digest)) def tearDown(self): self.tempfile.close() # Also deletes the temp file. # Remove the upload in `MEDIA_ROOT`. directory = os.path.join(settings.MEDIA_ROOT, '24') if os.path.exists(directory): shutil.rmtree(directory)
def create_photo_versions(sender, instance, **kwargs): """Create `PhotoVersion`` objects for the photo object defined by `instance`. A version is created for a bounding box defined by each PhotoSize instance. """ from photo.models import Photo, PhotoSize, PhotoVersion photo = instance ext = '.jpg' t = None try: pth = photo.image.path except NotImplementedError: from django.core.files.temp import NamedTemporaryFile t = NamedTemporaryFile(suffix=ext) ix = photo.image if ix.closed: # Reload from DB photo = Photo.objects.get(pk=photo.pk) ix = photo.image for d in ix.chunks(4000000): t.write(d) t.flush() t.seek(0) pth = t for size in PhotoSize.objects.all(): # Create a suitable filename. filename = '%s-%s-%s%s' % (photo.pk, uuid4().hex[::7], slugify(size.name)[:10], ext) ctype = guess_type(filename)[0] temp_file = TemporaryUploadedFile(name=filename, content_type=ctype, size=0, charset=None) if t: t.seek(0) try: version = PhotoVersion.objects.get(photo=photo, size=size) remove_model_image(version, 'image') version.image = None except PhotoVersion.DoesNotExist: version = PhotoVersion(photo=photo, size=size) if size.do_crop: resize_to, crop_box, input_image = get_perfect_fit_resize_crop(size.bounding_box, (photo.width, photo.height)) else: resize_to = size.bounding_box crop_box = None # Resize to a temporary location. resize(pth, resize_to, out_file_path=temp_file, crop=crop_box) # Save resized copy to `version` instance. temp_file.seek(0) # Prepare file for a re-read. version.image.save(name=filename, content=temp_file, save=True) temp_file.close() if t: t.close()
def create_customer_image(self, fp, anfexi): self.customer_image = self.create() # Process image (if the uploaded file has an image format) try: image = Image.open(fp) except IOError: # An IOError here means the file is not a recognized image format generic_file = TemporaryUploadedFile() extension = self._get_extension(fp) # Store the file as is and don't create a thumbnail self.customer_image.document.save( '{}-full.{}'.format( anfexi, extension ), ImageFile(generic_file) ) generic_file.close() return self.customer_image # Otherwise process images normally thumbnail, resize = self._process_image(image) # Write images on S3 thumbnail_file = self._make_temp_file(thumbnail.getvalue()) image_file = self._make_temp_file(resize.getvalue()) self.customer_image.thumbnail.save( '{}-thumbnail.png'.format(anfexi), ImageFile(thumbnail_file), save=False ) thumbnail_file.close() self.customer_image.document.save( '{}-full.png'.format(anfexi), ImageFile(image_file), save=False ) self.customer_image.save() image_file.close() return self.customer_image
def upload_chunked_file(request, param_name, allow_memory=True): """ Загрузчик файлов, переданных от форм. Поддерживает передачу файла по частям. Возвращает обертку над файлом (возможно в памяти), удаляющимся после закрытия. Если allow_memory = False, то мелкие файлы, которые Django сохраняет в память, будут принудительно сохранены во временные файлы на диске. Пример: from libs.upload import upload_chunked_file, NotLastChunk, TemporaryFileNotFoundError ... try: uploaded_file = upload_chunked_file(request, 'image') except TemporaryFileNotFoundError as e: return JsonResponse({ 'message': str(e), }, status=400) except NotLastChunk: return HttpResponse() request.user.avatar.save(uploaded_file.name, uploaded_file, save=False) uploaded_file.close() try: request.user.avatar.field.clean(request.user.avatar, request.user) except ValidationError as e: request.user.avatar.delete(save=False) return JsonResponse({ 'message': ', '.join(e.messages), }, status=400) request.user.avatar.clean() request.user.avatar.save() """ file = request.FILES[param_name] chunk_num = int(request.POST.get('chunk', 0)) chunk_count = int(request.POST.get('chunks', 1)) if chunk_count == 1: # файл одним куском if not isinstance(file, InMemoryUploadedFile): return file elif allow_memory: return file else: # принудительное сохранение в файл tmp = TemporaryUploadedFile(file.name, file.content_type, file.size, file.charset, file.content_type_extra) for chunk in file.chunks(): tmp.write(chunk) tmp.seek(0) tmp.flush() return tmp else: # pluploader отправляет имя "blob" file.name = os.path.basename(request.POST.get('name', file.name)) # генерируем имя, которое можно восстановить при получении # следующих чанков name, ext = os.path.splitext(file.name) hashname = '%s.%s' % (request.META.get('REMOTE_ADDR'), name) hashname = hashlib.md5(hashname.encode()).hexdigest() tempfile_name = '%s.upload%s' % (hashname, ext) tempfile_path = os.path.join(tempfile.gettempdir(), tempfile_name) if chunk_num > 0: if not os.path.exists(tempfile_path): raise TemporaryFileNotFoundError(_('Temporary file lost')) tmp = open(tempfile_path, 'ab+') if chunk_num == 0: tmp.seek(0) tmp.truncate() for chunk in file.chunks(): tmp.write(chunk) if chunk_num < chunk_count - 1: tmp.close() raise NotLastChunk(chunk_num + 1, chunk_count) tmp.seek(0) tmp.flush() file_info = os.stat(tempfile_path) return TempUploadedFile(tmp, name=file.name, content_type=file.content_type, size=file_info.st_size, charset=file.charset, content_type_extra=file.content_type_extra)
def test_datagroup_create(self): long_fn = "a filename that is too long " * 10 csv_string = ( "filename,title,document_type,url,organization\n" "0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,MS,, \n" f"{long_fn},Body Cream,MS,, \n") data = io.StringIO(csv_string) sample_csv = InMemoryUploadedFile( data, field_name="csv", name="register_records.csv", content_type="text/csv", size=len(csv_string), charset="utf-8", ) form_data = { "name": ["Walmart MSDS Test Group"], "description": ["test data group"], "group_type": ["2"], "downloaded_by": [str(User.objects.get(username="******").pk)], "downloaded_at": ["08/02/2018"], "download_script": ["1"], "data_source": ["10"], } request = self.factory.post(path="/datagroup/new/", data=form_data) request.FILES["csv"] = sample_csv request.user = User.objects.get(username="******") middleware = SessionMiddleware() middleware.process_request(request) request.session.save() middleware = MessageMiddleware() middleware.process_request(request) request.session.save() request.session = {} request.session["datasource_title"] = "Walmart" request.session["datasource_pk"] = 10 resp = views.data_group_create(request=request, pk=10) dg_exists = DataGroup.objects.filter( name="Walmart MSDS Test Group").exists() self.assertContains( resp, "filename: Ensure this value has at most 255 characters") self.assertFalse(dg_exists) csv_string = ( "filename,title,document_type,url,organization\n" "0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,MS,, \n" "0c68ab16-2065-4d9b-a8f2-e428eb192465.pdf,Body Cream,MS,, \n") data = io.StringIO(csv_string) sample_csv = InMemoryUploadedFile( data, field_name="csv", name="register_records.csv", content_type="text/csv", size=len(csv_string), charset="utf-8", ) request = self.factory.post(path="/datagroup/new", data=form_data) request.FILES["csv"] = sample_csv middleware = SessionMiddleware() middleware.process_request(request) request.session.save() middleware = MessageMiddleware() middleware.process_request(request) request.session.save() request.user = User.objects.get(username="******") request.session = {} request.session["datasource_title"] = "Walmart" request.session["datasource_pk"] = 10 resp = views.data_group_create(request=request, pk=10) self.assertEqual(resp.status_code, 302, "Should be redirecting") dg = DataGroup.objects.get(name="Walmart MSDS Test Group") self.assertEqual(f"/datagroup/{dg.pk}/", resp.url, "Should be redirecting to the proper URL") # test whether the file system folder was created self.assertIn( str(dg.fs_id), os.listdir(settings.MEDIA_ROOT), "The data group's UUID should be a folder in MEDIA_ROOT", ) # In the Data Group Detail Page resp = self.client.get(f"/datagroup/{dg.pk}/") # test whether the data documents were created docs = DataDocument.objects.filter(data_group=dg) self.assertEqual(len(docs), 2, "there should be two associated documents") # test whether the "Download Registered Records" link is like this example # <a href="/datagroup/a9c7f5a7-5ad4-4f75-b877-a3747f0cc081/download_registered_documents" class="btn btn-secondary"> csv_href = f"/datagroup/{dg.pk}/download_registered_documents/" self.assertIn( csv_href, str(resp._container), "The data group detail page must contain the right download link", ) # grab a filename from a data document and see if it's in the csv doc_fn = docs.first().filename # test whether the registered records csv download link works resp_rr_csv = self.client.get( csv_href) # this object should be of type StreamingHttpResponse docfound = "not found" for csv_row in resp_rr_csv.streaming_content: if doc_fn in str(csv_row): docfound = "found" self.assertEqual( docfound, "found", "the document file name should appear in the registered records csv", ) # Test whether the data document csv download works # URL on data group detail page: datagroup/docs_csv/{pk}/ dd_csv_href = (f"/datagroup/{dg.pk}/download_documents/" ) # this is an interpreted django URL resp_dd_csv = self.client.get(dd_csv_href) for csv_row in resp_dd_csv.streaming_content: if doc_fn in str(csv_row): docfound = "found" self.assertEqual( docfound, "found", "the document file name should appear in the data documents csv", ) # test uploading one pdf that matches a registered record doc = DataDocument.objects.filter(data_group_id=dg.pk).first() pdf = TemporaryUploadedFile(name=doc.filename, content_type="application/pdf", size=47, charset=None) request = self.factory.post(path="/datagroup/%s" % dg.pk, data={"uploaddocs-submit": "Submit"}) request.FILES["uploaddocs-documents"] = pdf middleware = SessionMiddleware() middleware.process_request(request) request.session.save() middleware = MessageMiddleware() middleware.process_request(request) request.session.save() request.user = User.objects.get(username="******") resp = views.data_group_detail(request=request, pk=dg.pk) pdf_path = f"{settings.MEDIA_ROOT}{dg.fs_id}/pdf/{doc.get_abstract_filename()}" self.assertTrue(os.path.exists(pdf_path), "the stored file should be in MEDIA_ROOT/dg.fs_id") pdf.close()
class UploadProgressCachedHandler(FileUploadHandler): """ Tracks progress for file uploads. The http post request must contain a header or query parameter, 'X-Progress-ID', which should contain a unique string to identify the upload to be tracked. Copied from: http://djangosnippets.org/snippets/678/ See views.py for upload_progress function... """ def __init__(self, request=None): super(UploadProgressCachedHandler, self).__init__(request) self.progress_id = None self.cache_key = None def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): """ """ self.content_length = content_length if 'X-Progress-ID' in self.request.GET: self.progress_id = self.request.GET['X-Progress-ID'] elif 'X-Progress-ID' in self.request.META: self.progress_id = self.request.META['X-Progress-ID'] if self.progress_id: self.cache_key = "%s_%s" % ( self.request.META['REMOTE_ADDR'], self.progress_id ) cache.set(self.cache_key, { 'length': self.content_length, 'uploaded': 0 }) def new_file(self, field_name, file_name, content_type, content_length, charset=None): """ """ self.field_name = field_name self.file_name = file_name self.content_type = content_type self.content_length = content_length self.charset = charset self.file = TemporaryUploadedFile( self.file_name, self.content_type, 0, self.charset ) def receive_data_chunk(self, raw_data, start): """ """ if self.cache_key: data = cache.get(self.cache_key) data['uploaded'] += self.chunk_size cache.set(self.cache_key, data) self.file.write(raw_data) return raw_data def file_complete(self, file_size): """ """ self.file.seek(0) self.file.size = file_size self.file.close() return self.file def upload_complete(self): """ """ if self.cache_key: cache.delete(self.cache_key)
def test_datagroup_create(self): long_fn = 'a filename that is too long ' * 10 csv_string = ( "filename,title,document_type,url,organization\n" "0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,UN,, \n" f"{long_fn},Body Cream,1,, \n") data = io.StringIO(csv_string) sample_csv = InMemoryUploadedFile(data, field_name='csv', name='register_records.csv', content_type='text/csv', size=len(csv_string), charset='utf-8') form_data = { 'name': ['Walmart MSDS Test Group'], 'description': ['test data group'], 'group_type': ['1'], 'downloaded_by': [str(User.objects.get(username='******').pk)], 'downloaded_at': ['08/02/2018'], 'download_script': ['1'], 'data_source': ['10'] } request = self.factory.post(path='/datagroup/new/', data=form_data) request.FILES['csv'] = sample_csv request.user = User.objects.get(username='******') request.session = {} request.session['datasource_title'] = 'Walmart' request.session['datasource_pk'] = 10 resp = views.data_group_create(request=request, pk=10) # print(resp.content) dg_exists = DataGroup.objects.filter( name='Walmart MSDS Test Group').exists() self.assertContains(resp, 'Filename too long') self.assertFalse(dg_exists, ) # print(dg.__dict__) csv_string = ( "filename,title,document_type,url,organization\n" "0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,UN,, \n" "0c68ab16-2065-4d9b-a8f2-e428eb192465.pdf,Body Cream,UN,, \n") data = io.StringIO(csv_string) sample_csv = InMemoryUploadedFile(data, field_name='csv', name='register_records.csv', content_type='text/csv', size=len(csv_string), charset='utf-8') request = self.factory.post(path='/datagroup/new', data=form_data) request.FILES['csv'] = sample_csv request.user = User.objects.get(username='******') request.session = {} request.session['datasource_title'] = 'Walmart' request.session['datasource_pk'] = 10 resp = views.data_group_create(request=request, pk=10) self.assertEqual(resp.status_code, 302, "Should be redirecting") dg = DataGroup.objects.get(name='Walmart MSDS Test Group') self.assertEqual(f'/datagroup/{dg.pk}/', resp.url, "Should be redirecting to the proper URL") # test whether the file system folder was created self.assertIn( str(dg.fs_id), os.listdir(settings.MEDIA_ROOT), "The data group's UUID should be a folder in MEDIA_ROOT") # In the Data Group Detail Page resp = self.client.get(f'/datagroup/{dg.pk}/') # test whether the data documents were created docs = DataDocument.objects.filter(data_group=dg) self.assertEqual(len(docs), 2, "there should be two associated documents") # test whether the "Download Registered Records" link is like this example # <a href="/datagroup/a9c7f5a7-5ad4-4f75-b877-a3747f0cc081/registered_records.csv" class="btn btn-secondary"> # <span class="oi oi-spreadsheet"></span> Download Registered Records CSV</a> csv_href = f'/datagroup/{dg.pk}/registered_records.csv' self.assertIn( csv_href, str(resp._container), "The data group detail page must contain the right download link") # grab a filename from a data document and see if it's in the csv doc_fn = docs.first().filename # test whether the registered records csv download link works resp_rr_csv = self.client.get( csv_href) # this object should be of type StreamingHttpResponse docfound = 'not found' for csv_row in resp_rr_csv.streaming_content: if doc_fn in str(csv_row): docfound = 'found' self.assertEqual( docfound, 'found', "the document file name should appear in the registered records csv" ) # Test whether the data document csv download works # URL on data group detail page: datagroup/docs_csv/{pk}/ dd_csv_href = f'/datagroup/docs_csv/{dg.pk}/' # this is an interpreted django URL resp_dd_csv = self.client.get(dd_csv_href) for csv_row in resp_dd_csv.streaming_content: if doc_fn in str(csv_row): docfound = 'found' self.assertEqual( docfound, 'found', "the document file name should appear in the data documents csv") # test whether the "Download All PDF Documents" link works dg_zip_href = f'/datagroup/pdfs_zipped/{dg.pk}/' # this is the django-interpreted URL self.assertIn( dg_zip_href, str(resp._container), "The data group detail page must contain the right zip download link" ) resp_zip = self.client.get(dg_zip_href) # test uploading one pdf that matches a registered record f = TemporaryUploadedFile( name='0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf', content_type='application/pdf', size=47, charset=None) request = self.factory.post(path='/datagroup/%s' % dg.pk, data={'upload': 'Submit'}) request.FILES['multifiles'] = f request.user = User.objects.get(username='******') resp = views.data_group_detail(request=request, pk=dg.pk) doc = DataDocument.objects.get(title='NUTRA NAIL') fn = doc.get_abstract_filename() folder_name = str(dg.fs_id) stored_file = f'{folder_name}/pdf/{fn}' pdf_path = f'{settings.MEDIA_ROOT}{stored_file}' self.assertTrue(os.path.exists(pdf_path), "the stored file should be in MEDIA_ROOT/dg.fs_id") f.close()