def _make_tempfile(self, filename, content):
fileobj = TemporaryUploadedFile(
name=filename + ".tempfile",
content_type='text/plain',
size=0,
charset='utf8',
)
fileobj.write(content)
fileobj.flush()
return fileobj
def _make_tempfile(self, filename, content):
fileobj = TemporaryUploadedFile(
name=filename + ".tempfile",
content_type='text/plain',
size=0,
charset='utf8',
)
fileobj.write(content)
fileobj.flush()
return fileobj
def upload_file(url, timeout=5):
"""
Загрузка файла по урлу во временный файл.
Пример:
from libs.upload import *
...
try:
uploaded_file = upload_file('http://host.ru/image.jpg')
except URLError as e:
return JsonResponse({
'message': str(e.msg),
}, status=e.code)
request.user.avatar.save(uploaded_file.name, uploaded_file, save=False)
uploaded_file.close()
try:
request.user.full_clean()
except ValidationError as e:
request.user.avatar.delete(save=False)
return JsonResponse({
'message': ', '.join(e.messages),
}, status=400)
else:
request.user.save()
"""
logger.debug('Uploading %s...', url)
with contextlib.closing(urlopen(url, timeout=timeout)) as fp:
headers = fp.info()
file_name = url.split('/')[-1]
content_type = headers.get('content-type')
file_size = headers.get('content-length')
charset = 'utf-8'
tmp = TemporaryUploadedFile(file_name, content_type, file_size,
charset, {})
while True:
block = fp.read(8 * 1024)
if not block:
break
tmp.write(block)
logger.debug('Uploaded %s to file %s', url, tmp.file.name)
tmp.seek(0)
tmp.flush()
return tmp
def write(self):
self.name = uuid.uuid4().hex
img_tmp = TemporaryUploadedFile(self.name, self.mimetype, self.size,
None)
img_tmp.write(self._data)
img_tmp.flush()
# use temporary file size if there is no content-length in response header
# The file size is validated at converter
if self.size == 0:
img_tmp.size = os.path.getsize(img_tmp.temporary_file_path())
self._data = None
return img_tmp
def test_file_context(self):
response = None
with open("threats/test_data/boss.gif", "rb") as boss_reader:
file_data = boss_reader.read()
boss_reader.seek(0)
response = self.client.post('/', {
'artifact': '{"type": "file.content"}',
'file': boss_reader
},
format="multipart")
upload_file_args = {
"name": "boss.gif",
"content_type": "application/octet-stream",
"size": 29927,
"charset": None,
}
self.assertEqual(response.status_code, 200)
temp_file = TemporaryUploadedFile(**upload_file_args)
temp_file.write(file_data)
temp_file.flush()
context = SearchContext({"type": "file.content", "value": temp_file})
context.save()
self.assertEqual(context.file_data_len, len(file_data))
loaded_context = SearchContext.load(context.id)
self.assertEqual(loaded_context.base64_file_data_len,
context.base64_file_data_len)
self.assertEqual(loaded_context.file_data_len, context.file_data_len)
with open(loaded_context.value.temporary_file_path(),
"rb") as temp_file:
loaded_file_data = temp_file.read()
for counter in range(0, len(loaded_file_data) // 100):
begin = counter * 100
end = begin + 100
self.assertEqual(file_data[begin:end], loaded_file_data[begin:end])
self.assertEqual(len(file_data), len(loaded_file_data))
def __setstate__(self, pickle_dictionary):
if pickle_dictionary["_context"]["type"] == "file.content" and \
isinstance(pickle_dictionary["_context"]["value"], dict):
arguments = pickle_dictionary["_context"]["value"]
# File data info, especially useful for test pickling
self.base64_file_data_len = len(arguments["content"])
file_content = base64.b64decode(arguments.pop("content"))
# File data info, especially useful for test pickling
self.file_data_len = len(file_content)
file_object = TemporaryUploadedFile(**arguments)
file_object.write(file_content)
file_object.flush()
pickle_dictionary["_context"]["value"] = file_object
self.__dict__.update(pickle_dictionary)
def ensure_saved(self, file):
"""This may create a temporary file, which will be deleted when
it's closed, so always close() it but only when you've finished!"""
if isinstance(file, InMemoryUploadedFile):
print "Writing %s to disk (%d bytes)" % (file, file.size)
tmp = TemporaryUploadedFile(name=file.name,
content_type=file.content_type, size=file.size,
charset=file.charset)
file.seek(0)
buf = file.read()
tmp.write(buf)
print "Wrote %d bytes" % len(buf)
tmp.flush()
else:
tmp = file
if isinstance(tmp, TemporaryUploadedFile):
path = tmp.temporary_file_path()
else:
path = tmp.name
return (tmp, path)
def upload_chunked_file(request, param_name, allow_memory=True):
"""
Загрузчик файлов, переданных от форм.
Поддерживает передачу файла по частям.
Возвращает обертку над файлом (возможно в памяти), удаляющимся после закрытия.
Если allow_memory = False, то мелкие файлы, которые Django сохраняет в память,
будут принудительно сохранены во временные файлы на диске.
Пример:
from libs.upload import upload_chunked_file, NotLastChunk, TemporaryFileNotFoundError
...
try:
uploaded_file = upload_chunked_file(request, 'image')
except TemporaryFileNotFoundError as e:
return JsonResponse({
'message': str(e),
}, status=400)
except NotLastChunk:
return HttpResponse()
request.user.avatar.save(uploaded_file.name, uploaded_file, save=False)
uploaded_file.close()
try:
request.user.avatar.field.clean(request.user.avatar, request.user)
except ValidationError as e:
request.user.avatar.delete(save=False)
return JsonResponse({
'message': ', '.join(e.messages),
}, status=400)
request.user.avatar.clean()
request.user.avatar.save()
"""
file = request.FILES[param_name]
chunk_num = int(request.POST.get('chunk', 0))
chunk_count = int(request.POST.get('chunks', 1))
if chunk_count == 1:
# файл одним куском
if not isinstance(file, InMemoryUploadedFile):
return file
elif allow_memory:
return file
else:
# принудительное сохранение в файл
tmp = TemporaryUploadedFile(file.name, file.content_type,
file.size, file.charset,
file.content_type_extra)
for chunk in file.chunks():
tmp.write(chunk)
tmp.seek(0)
tmp.flush()
return tmp
else:
# pluploader отправляет имя "blob"
file.name = os.path.basename(request.POST.get('name', file.name))
# генерируем имя, которое можно восстановить при получении
# следующих чанков
name, ext = os.path.splitext(file.name)
hashname = '%s.%s' % (request.META.get('REMOTE_ADDR'), name)
hashname = hashlib.md5(hashname.encode()).hexdigest()
tempfile_name = '%s.upload%s' % (hashname, ext)
tempfile_path = os.path.join(tempfile.gettempdir(), tempfile_name)
if chunk_num > 0:
if not os.path.exists(tempfile_path):
raise TemporaryFileNotFoundError(_('Temporary file lost'))
tmp = open(tempfile_path, 'ab+')
if chunk_num == 0:
tmp.seek(0)
tmp.truncate()
for chunk in file.chunks():
tmp.write(chunk)
if chunk_num < chunk_count - 1:
tmp.close()
raise NotLastChunk(chunk_num + 1, chunk_count)
tmp.seek(0)
tmp.flush()
file_info = os.stat(tempfile_path)
return TempUploadedFile(tmp,
name=file.name,
content_type=file.content_type,
size=file_info.st_size,
charset=file.charset,
content_type_extra=file.content_type_extra)
def _get_documents_from_api(self) -> List[Tuple[str, BytesIO]]:
"""Retrieve the documents and their content from the Documenten API."""
logger.debug("Retrieving documents from Documenten API")
variables = self.task.get_variables()
document_urls = check_variable(variables, "documents")
client_pool = DRCClientPool(variables)
client_pool.populate_clients(self.task, document_urls)
documents = []
current_total_documents_size = 0
for document_url in document_urls:
# Getting the appropriate client
document_client = client_pool.get_client_for(document_url)
# Retrieving the document
document_data = document_client.retrieve(
resource="enkelvoudiginformatieobject",
url=document_url,
)
# Retrieving the content of the document
# Need use requests directly instead of `document_client.request()` since the response is not in JSON format
response = requests.get(
document_data["inhoud"],
headers=document_client.auth_header,
stream=True,
)
# Get the document size in bytes
document_size = document_data["bestandsomvang"]
# If the size of the document is above the max size or if all the documents together have already reached
# the maximum size, write the file content to a temporary file
if (document_size > settings.MAX_DOCUMENT_SIZE
or (current_total_documents_size + document_size) >
settings.MAX_TOTAL_DOCUMENT_SIZE):
# The file is created with rb+ mode by default
tmp_file_object = TemporaryUploadedFile(
name=
f"{document_data['titel']}-{get_random_string(length=5)}.tempfile",
content_type="application/octet-stream",
size=document_size,
charset=
None, # Required argument in TemporaryUploadedFile, but not in parent class UploadedFile
)
for chunk in response.iter_content(
chunk_size=settings.CHUNK_SIZE):
tmp_file_object.write(chunk)
tmp_file_object.flush()
doc_tuple = (document_data["titel"], tmp_file_object)
else:
doc_tuple = (document_data["titel"], BytesIO(response.content))
current_total_documents_size += document_size
response.close()
documents.append(doc_tuple)
return documents
