class TemporaryFileUploadHandler(FileUploadHandler):
"""
Upload handler that streams data into a temporary file.
"""
def new_file(self, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
"""
super().new_file(*args, **kwargs)
self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0,
self.charset,
self.content_type_extra)
def receive_data_chunk(self, raw_data, start):
self.file.write(raw_data)
def file_complete(self, file_size):
self.file.seek(0)
self.file.size = file_size
return self.file
def upload_interrupted(self):
if hasattr(self, 'file'):
temp_location = self.file.temporary_file_path()
try:
self.file.close()
os.remove(temp_location)
except FileNotFoundError:
pass
class FileWriteTest(TestCase):
def setUp(self):
self.text = "Spam Spam Spam Spam.\n"
self.digest = hashlib.sha1(self.text).hexdigest()
self.tempfile = TemporaryUploadedFile('spam4.txt', 'text/plain',
len(self.text), 'utf-8')
self.tempfile.file.write(self.text)
self.tempfile.file.seek(0)
def test(self):
new_upload = Upload(file=self.tempfile)
new_upload.save()
# Upload has been saved to the database.
self.assert_(new_upload.pk)
# Upload contains correct content.
self.assertEqual(new_upload.file.read(), self.text)
# Filename is the hash of the file contents.
self.assert_(new_upload.file.name.startswith(self.digest))
def tearDown(self):
self.tempfile.close() # Also deletes the temp file.
# Remove the upload in `MEDIA_ROOT`.
directory = os.path.join(settings.MEDIA_ROOT, '24')
if os.path.exists(directory):
shutil.rmtree(directory)
def create_photo_versions(sender, instance, **kwargs):
"""Create `PhotoVersion`` objects for the photo object defined by `instance`.
A version is created for a bounding box defined by each PhotoSize instance.
"""
from photo.models import Photo, PhotoSize, PhotoVersion
photo = instance
ext = '.jpg'
t = None
try:
pth = photo.image.path
except NotImplementedError:
from django.core.files.temp import NamedTemporaryFile
t = NamedTemporaryFile(suffix=ext)
ix = photo.image
if ix.closed:
# Reload from DB
photo = Photo.objects.get(pk=photo.pk)
ix = photo.image
for d in ix.chunks(4000000):
t.write(d)
t.flush()
t.seek(0)
pth = t
for size in PhotoSize.objects.all():
# Create a suitable filename.
filename = '%s-%s-%s%s' % (photo.pk, uuid4().hex[::7], slugify(size.name)[:10], ext)
ctype = guess_type(filename)[0]
temp_file = TemporaryUploadedFile(name=filename, content_type=ctype, size=0, charset=None)
if t:
t.seek(0)
try:
version = PhotoVersion.objects.get(photo=photo, size=size)
remove_model_image(version, 'image')
version.image = None
except PhotoVersion.DoesNotExist:
version = PhotoVersion(photo=photo, size=size)
if size.do_crop:
resize_to, crop_box, input_image = get_perfect_fit_resize_crop(size.bounding_box, (photo.width, photo.height))
else:
resize_to = size.bounding_box
crop_box = None
# Resize to a temporary location.
resize(pth, resize_to, out_file_path=temp_file, crop=crop_box)
# Save resized copy to `version` instance.
temp_file.seek(0) # Prepare file for a re-read.
version.image.save(name=filename, content=temp_file, save=True)
temp_file.close()
if t:
t.close()
def create_customer_image(self, fp, anfexi):
self.customer_image = self.create()
# Process image (if the uploaded file has an image format)
try:
image = Image.open(fp)
except IOError:
# An IOError here means the file is not a recognized image format
generic_file = TemporaryUploadedFile()
extension = self._get_extension(fp)
# Store the file as is and don't create a thumbnail
self.customer_image.document.save(
'{}-full.{}'.format(
anfexi, extension
),
ImageFile(generic_file)
)
generic_file.close()
return self.customer_image
# Otherwise process images normally
thumbnail, resize = self._process_image(image)
# Write images on S3
thumbnail_file = self._make_temp_file(thumbnail.getvalue())
image_file = self._make_temp_file(resize.getvalue())
self.customer_image.thumbnail.save(
'{}-thumbnail.png'.format(anfexi),
ImageFile(thumbnail_file), save=False
)
thumbnail_file.close()
self.customer_image.document.save(
'{}-full.png'.format(anfexi),
ImageFile(image_file), save=False
)
self.customer_image.save()
image_file.close()
return self.customer_image
def upload_chunked_file(request, param_name, allow_memory=True):
"""
Загрузчик файлов, переданных от форм.
Поддерживает передачу файла по частям.
Возвращает обертку над файлом (возможно в памяти), удаляющимся после закрытия.
Если allow_memory = False, то мелкие файлы, которые Django сохраняет в память,
будут принудительно сохранены во временные файлы на диске.
Пример:
from libs.upload import upload_chunked_file, NotLastChunk, TemporaryFileNotFoundError
...
try:
uploaded_file = upload_chunked_file(request, 'image')
except TemporaryFileNotFoundError as e:
return JsonResponse({
'message': str(e),
}, status=400)
except NotLastChunk:
return HttpResponse()
request.user.avatar.save(uploaded_file.name, uploaded_file, save=False)
uploaded_file.close()
try:
request.user.avatar.field.clean(request.user.avatar, request.user)
except ValidationError as e:
request.user.avatar.delete(save=False)
return JsonResponse({
'message': ', '.join(e.messages),
}, status=400)
request.user.avatar.clean()
request.user.avatar.save()
"""
file = request.FILES[param_name]
chunk_num = int(request.POST.get('chunk', 0))
chunk_count = int(request.POST.get('chunks', 1))
if chunk_count == 1:
# файл одним куском
if not isinstance(file, InMemoryUploadedFile):
return file
elif allow_memory:
return file
else:
# принудительное сохранение в файл
tmp = TemporaryUploadedFile(file.name, file.content_type,
file.size, file.charset,
file.content_type_extra)
for chunk in file.chunks():
tmp.write(chunk)
tmp.seek(0)
tmp.flush()
return tmp
else:
# pluploader отправляет имя "blob"
file.name = os.path.basename(request.POST.get('name', file.name))
# генерируем имя, которое можно восстановить при получении
# следующих чанков
name, ext = os.path.splitext(file.name)
hashname = '%s.%s' % (request.META.get('REMOTE_ADDR'), name)
hashname = hashlib.md5(hashname.encode()).hexdigest()
tempfile_name = '%s.upload%s' % (hashname, ext)
tempfile_path = os.path.join(tempfile.gettempdir(), tempfile_name)
if chunk_num > 0:
if not os.path.exists(tempfile_path):
raise TemporaryFileNotFoundError(_('Temporary file lost'))
tmp = open(tempfile_path, 'ab+')
if chunk_num == 0:
tmp.seek(0)
tmp.truncate()
for chunk in file.chunks():
tmp.write(chunk)
if chunk_num < chunk_count - 1:
tmp.close()
raise NotLastChunk(chunk_num + 1, chunk_count)
tmp.seek(0)
tmp.flush()
file_info = os.stat(tempfile_path)
return TempUploadedFile(tmp,
name=file.name,
content_type=file.content_type,
size=file_info.st_size,
charset=file.charset,
content_type_extra=file.content_type_extra)
def test_datagroup_create(self):
long_fn = "a filename that is too long " * 10
csv_string = (
"filename,title,document_type,url,organization\n"
"0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,MS,, \n"
f"{long_fn},Body Cream,MS,, \n")
data = io.StringIO(csv_string)
sample_csv = InMemoryUploadedFile(
data,
field_name="csv",
name="register_records.csv",
content_type="text/csv",
size=len(csv_string),
charset="utf-8",
)
form_data = {
"name": ["Walmart MSDS Test Group"],
"description": ["test data group"],
"group_type": ["2"],
"downloaded_by": [str(User.objects.get(username="******").pk)],
"downloaded_at": ["08/02/2018"],
"download_script": ["1"],
"data_source": ["10"],
}
request = self.factory.post(path="/datagroup/new/", data=form_data)
request.FILES["csv"] = sample_csv
request.user = User.objects.get(username="******")
middleware = SessionMiddleware()
middleware.process_request(request)
request.session.save()
middleware = MessageMiddleware()
middleware.process_request(request)
request.session.save()
request.session = {}
request.session["datasource_title"] = "Walmart"
request.session["datasource_pk"] = 10
resp = views.data_group_create(request=request, pk=10)
dg_exists = DataGroup.objects.filter(
name="Walmart MSDS Test Group").exists()
self.assertContains(
resp, "filename: Ensure this value has at most 255 characters")
self.assertFalse(dg_exists)
csv_string = (
"filename,title,document_type,url,organization\n"
"0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,MS,, \n"
"0c68ab16-2065-4d9b-a8f2-e428eb192465.pdf,Body Cream,MS,, \n")
data = io.StringIO(csv_string)
sample_csv = InMemoryUploadedFile(
data,
field_name="csv",
name="register_records.csv",
content_type="text/csv",
size=len(csv_string),
charset="utf-8",
)
request = self.factory.post(path="/datagroup/new", data=form_data)
request.FILES["csv"] = sample_csv
middleware = SessionMiddleware()
middleware.process_request(request)
request.session.save()
middleware = MessageMiddleware()
middleware.process_request(request)
request.session.save()
request.user = User.objects.get(username="******")
request.session = {}
request.session["datasource_title"] = "Walmart"
request.session["datasource_pk"] = 10
resp = views.data_group_create(request=request, pk=10)
self.assertEqual(resp.status_code, 302, "Should be redirecting")
dg = DataGroup.objects.get(name="Walmart MSDS Test Group")
self.assertEqual(f"/datagroup/{dg.pk}/", resp.url,
"Should be redirecting to the proper URL")
# test whether the file system folder was created
self.assertIn(
str(dg.fs_id),
os.listdir(settings.MEDIA_ROOT),
"The data group's UUID should be a folder in MEDIA_ROOT",
)
# In the Data Group Detail Page
resp = self.client.get(f"/datagroup/{dg.pk}/")
# test whether the data documents were created
docs = DataDocument.objects.filter(data_group=dg)
self.assertEqual(len(docs), 2,
"there should be two associated documents")
# test whether the "Download Registered Records" link is like this example
# <a href="/datagroup/a9c7f5a7-5ad4-4f75-b877-a3747f0cc081/download_registered_documents" class="btn btn-secondary">
csv_href = f"/datagroup/{dg.pk}/download_registered_documents/"
self.assertIn(
csv_href,
str(resp._container),
"The data group detail page must contain the right download link",
)
# grab a filename from a data document and see if it's in the csv
doc_fn = docs.first().filename
# test whether the registered records csv download link works
resp_rr_csv = self.client.get(
csv_href) # this object should be of type StreamingHttpResponse
docfound = "not found"
for csv_row in resp_rr_csv.streaming_content:
if doc_fn in str(csv_row):
docfound = "found"
self.assertEqual(
docfound,
"found",
"the document file name should appear in the registered records csv",
)
# Test whether the data document csv download works
# URL on data group detail page: datagroup/docs_csv/{pk}/
dd_csv_href = (f"/datagroup/{dg.pk}/download_documents/"
) # this is an interpreted django URL
resp_dd_csv = self.client.get(dd_csv_href)
for csv_row in resp_dd_csv.streaming_content:
if doc_fn in str(csv_row):
docfound = "found"
self.assertEqual(
docfound,
"found",
"the document file name should appear in the data documents csv",
)
# test uploading one pdf that matches a registered record
doc = DataDocument.objects.filter(data_group_id=dg.pk).first()
pdf = TemporaryUploadedFile(name=doc.filename,
content_type="application/pdf",
size=47,
charset=None)
request = self.factory.post(path="/datagroup/%s" % dg.pk,
data={"uploaddocs-submit": "Submit"})
request.FILES["uploaddocs-documents"] = pdf
middleware = SessionMiddleware()
middleware.process_request(request)
request.session.save()
middleware = MessageMiddleware()
middleware.process_request(request)
request.session.save()
request.user = User.objects.get(username="******")
resp = views.data_group_detail(request=request, pk=dg.pk)
pdf_path = f"{settings.MEDIA_ROOT}{dg.fs_id}/pdf/{doc.get_abstract_filename()}"
self.assertTrue(os.path.exists(pdf_path),
"the stored file should be in MEDIA_ROOT/dg.fs_id")
pdf.close()
class UploadProgressCachedHandler(FileUploadHandler):
"""
Tracks progress for file uploads.
The http post request must contain a header or query parameter,
'X-Progress-ID', which should contain a unique string to identify
the upload to be tracked.
Copied from:
http://djangosnippets.org/snippets/678/
See views.py for upload_progress function...
"""
def __init__(self, request=None):
super(UploadProgressCachedHandler, self).__init__(request)
self.progress_id = None
self.cache_key = None
def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
"""
"""
self.content_length = content_length
if 'X-Progress-ID' in self.request.GET:
self.progress_id = self.request.GET['X-Progress-ID']
elif 'X-Progress-ID' in self.request.META:
self.progress_id = self.request.META['X-Progress-ID']
if self.progress_id:
self.cache_key = "%s_%s" % (
self.request.META['REMOTE_ADDR'],
self.progress_id
)
cache.set(self.cache_key, {
'length': self.content_length,
'uploaded': 0
})
def new_file(self, field_name, file_name, content_type, content_length, charset=None):
"""
"""
self.field_name = field_name
self.file_name = file_name
self.content_type = content_type
self.content_length = content_length
self.charset = charset
self.file = TemporaryUploadedFile(
self.file_name,
self.content_type,
0,
self.charset
)
def receive_data_chunk(self, raw_data, start):
"""
"""
if self.cache_key:
data = cache.get(self.cache_key)
data['uploaded'] += self.chunk_size
cache.set(self.cache_key, data)
self.file.write(raw_data)
return raw_data
def file_complete(self, file_size):
"""
"""
self.file.seek(0)
self.file.size = file_size
self.file.close()
return self.file
def upload_complete(self):
"""
"""
if self.cache_key:
cache.delete(self.cache_key)
def test_datagroup_create(self):
long_fn = 'a filename that is too long ' * 10
csv_string = (
"filename,title,document_type,url,organization\n"
"0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,UN,, \n"
f"{long_fn},Body Cream,1,, \n")
data = io.StringIO(csv_string)
sample_csv = InMemoryUploadedFile(data,
field_name='csv',
name='register_records.csv',
content_type='text/csv',
size=len(csv_string),
charset='utf-8')
form_data = {
'name': ['Walmart MSDS Test Group'],
'description': ['test data group'],
'group_type': ['1'],
'downloaded_by': [str(User.objects.get(username='******').pk)],
'downloaded_at': ['08/02/2018'],
'download_script': ['1'],
'data_source': ['10']
}
request = self.factory.post(path='/datagroup/new/', data=form_data)
request.FILES['csv'] = sample_csv
request.user = User.objects.get(username='******')
request.session = {}
request.session['datasource_title'] = 'Walmart'
request.session['datasource_pk'] = 10
resp = views.data_group_create(request=request, pk=10)
# print(resp.content)
dg_exists = DataGroup.objects.filter(
name='Walmart MSDS Test Group').exists()
self.assertContains(resp, 'Filename too long')
self.assertFalse(dg_exists, )
# print(dg.__dict__)
csv_string = (
"filename,title,document_type,url,organization\n"
"0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf,NUTRA NAIL,UN,, \n"
"0c68ab16-2065-4d9b-a8f2-e428eb192465.pdf,Body Cream,UN,, \n")
data = io.StringIO(csv_string)
sample_csv = InMemoryUploadedFile(data,
field_name='csv',
name='register_records.csv',
content_type='text/csv',
size=len(csv_string),
charset='utf-8')
request = self.factory.post(path='/datagroup/new', data=form_data)
request.FILES['csv'] = sample_csv
request.user = User.objects.get(username='******')
request.session = {}
request.session['datasource_title'] = 'Walmart'
request.session['datasource_pk'] = 10
resp = views.data_group_create(request=request, pk=10)
self.assertEqual(resp.status_code, 302, "Should be redirecting")
dg = DataGroup.objects.get(name='Walmart MSDS Test Group')
self.assertEqual(f'/datagroup/{dg.pk}/', resp.url,
"Should be redirecting to the proper URL")
# test whether the file system folder was created
self.assertIn(
str(dg.fs_id), os.listdir(settings.MEDIA_ROOT),
"The data group's UUID should be a folder in MEDIA_ROOT")
# In the Data Group Detail Page
resp = self.client.get(f'/datagroup/{dg.pk}/')
# test whether the data documents were created
docs = DataDocument.objects.filter(data_group=dg)
self.assertEqual(len(docs), 2,
"there should be two associated documents")
# test whether the "Download Registered Records" link is like this example
# <a href="/datagroup/a9c7f5a7-5ad4-4f75-b877-a3747f0cc081/registered_records.csv" class="btn btn-secondary">
# <span class="oi oi-spreadsheet"></span> Download Registered Records CSV</a>
csv_href = f'/datagroup/{dg.pk}/registered_records.csv'
self.assertIn(
csv_href, str(resp._container),
"The data group detail page must contain the right download link")
# grab a filename from a data document and see if it's in the csv
doc_fn = docs.first().filename
# test whether the registered records csv download link works
resp_rr_csv = self.client.get(
csv_href) # this object should be of type StreamingHttpResponse
docfound = 'not found'
for csv_row in resp_rr_csv.streaming_content:
if doc_fn in str(csv_row):
docfound = 'found'
self.assertEqual(
docfound, 'found',
"the document file name should appear in the registered records csv"
)
# Test whether the data document csv download works
# URL on data group detail page: datagroup/docs_csv/{pk}/
dd_csv_href = f'/datagroup/docs_csv/{dg.pk}/' # this is an interpreted django URL
resp_dd_csv = self.client.get(dd_csv_href)
for csv_row in resp_dd_csv.streaming_content:
if doc_fn in str(csv_row):
docfound = 'found'
self.assertEqual(
docfound, 'found',
"the document file name should appear in the data documents csv")
# test whether the "Download All PDF Documents" link works
dg_zip_href = f'/datagroup/pdfs_zipped/{dg.pk}/' # this is the django-interpreted URL
self.assertIn(
dg_zip_href, str(resp._container),
"The data group detail page must contain the right zip download link"
)
resp_zip = self.client.get(dg_zip_href)
# test uploading one pdf that matches a registered record
f = TemporaryUploadedFile(
name='0bf5755e-3a08-4024-9d2f-0ea155a9bd17.pdf',
content_type='application/pdf',
size=47,
charset=None)
request = self.factory.post(path='/datagroup/%s' % dg.pk,
data={'upload': 'Submit'})
request.FILES['multifiles'] = f
request.user = User.objects.get(username='******')
resp = views.data_group_detail(request=request, pk=dg.pk)
doc = DataDocument.objects.get(title='NUTRA NAIL')
fn = doc.get_abstract_filename()
folder_name = str(dg.fs_id)
stored_file = f'{folder_name}/pdf/{fn}'
pdf_path = f'{settings.MEDIA_ROOT}{stored_file}'
self.assertTrue(os.path.exists(pdf_path),
"the stored file should be in MEDIA_ROOT/dg.fs_id")
f.close()
