def download_resource_file(url: str, target_name: str) -> str:
"""Download a resource file and store it using GeoNode's `storage_manager`.
Downloads use the django `UploadedFile` helper classes. Depending on the size of the
remote resource, we may download it into an in-memory buffer or store it on a
temporary location on disk. After having downloaded the file, we use `storage_manager`
to save it in the appropriate location.
"""
response = requests.get(url, stream=True)
response.raise_for_status()
file_size = response.headers.get("Content-Length")
content_type = response.headers.get("Content-Type")
charset = response.apparent_encoding
if file_size is not None and int(
file_size) < config.HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE:
logger.debug("Downloading to an in-memory buffer...")
file_ = uploadedfile.InMemoryUploadedFile(None, None, target_name,
content_type, file_size,
charset)
else:
logger.debug("Downloading to a temporary file...")
file_ = uploadedfile.TemporaryUploadedFile(target_name, content_type,
file_size, charset)
with file_.open("wb+") as fd:
for chunk in response.iter_content(chunk_size=None,
decode_unicode=False):
fd.write(chunk)
fd.seek(0)
result = storage_manager.save(target_name, fd)
return result
def inner(filename, content):
return uploadedfile.InMemoryUploadedFile(file=io.BytesIO(content),
field_name='test_field',
name='_save_new_file.txt',
content_type='text/plain',
size=0,
charset='utf8')
def test_path_is_mime(self):
"""
Check expected result from a query asking for objects tagged with a
binary blob of a certain MIME type (e.g. image/png).
"""
binary_tag = logic.create_tag(
user=self.site_admin_user,
name="bin-tag",
description="A tag for annotating binary data.",
type_of="a",
namespace=self.test_namespace,
private=False,
)
val = uploadedfile.InMemoryUploadedFile(
file=BytesIO(b"hello"),
field_name="",
name="file.txt",
content_type="text/text",
size=5,
charset="utf-8",
)
annotation = binary_tag.annotate(self.admin_user, "test_object1", val)
annotation.save()
tokens = list(
self.lexer.tokenize("test_namespace/bin-tag is mime:text/text"))
parser = query.QueryParser(self.admin_user, self.lexer.tag_paths)
result = parser.parse((x for x in tokens))
self.assertEqual(len(result), 1)
self.assertIn("test_object1", result)
def make_file(func, my_object, elements):
new_elements = func(elements).sorted()
buf = sorting_classes.write_file(new_elements)
buf.seek(0, 2)
result_file = uploadedfile.InMemoryUploadedFile(
buf, 'result', f'file{my_object.pk}.txt', None, buf.tell(), None)
my_object.result.save(result_file.name, result_file)
def _make_memfile(self, filename, content):
return uploadedfile.InMemoryUploadedFile(
file=six.BytesIO(content),
field_name='test_field',
name='_save_new_file.txt',
content_type='text/plain',
size=0,
charset='utf8',
)
def create_in_memory_image(image, name, size):
"""
Resizes the image and saves it as InMemoryUploadedFile object
Returns the InMemoryUploadedFile object with the image data
"""
output = io.BytesIO() # create an io object
# resize the image and save it to the io object
image_resize(image, output, size)
# get MIME type of the image
mime = magic.from_buffer(output.getvalue(), mime=True)
# create InMemoryUploadedFile using data from the io
return uploadedfile.InMemoryUploadedFile(output, 'ImageField', name, mime,
sys.getsizeof(output), None)
def make_thumbnail(self, thumb):
size = 128, 128
thumb.thumbnail(size, Image.ANTIALIAS)
# convert to greyscale?
# thumb = thumb.convert('L')
thumb_io = io.BytesIO()
thumb.save(thumb_io, format='JPEG')
thumb_file = uploadedfile.InMemoryUploadedFile(
file=thumb_io,
field_name=None,
name=os.path.split(self.image.name)[-1],
content_type='image/jpeg',
size=thumb_io.getbuffer().nbytes,
charset=None)
return thumb_file
def perform_image_crop(image_obj, crop_rect=None):
img_ext = os.path.splitext(image_obj.name)[1][1:].upper()
img_ext = 'JPEG' if img_ext == 'JPG' else img_ext
if crop_rect is None:
return image_obj
image = BytesIO(image_obj.read())
base_image = Image.open(image)
tmp_img, tmp_file = base_image.crop(crop_rect), BytesIO()
tmp_img.save(tmp_file, format=img_ext)
tmp_file = ContentFile(tmp_file.getvalue())
return uploadedfile.InMemoryUploadedFile(tmp_file, None, image_obj.name,
image_obj.content_type,
tmp_file.tell, None)
def put(request):
if not request.user.is_authenticated:
raise exceptions.PermissionDenied()
uploaded = uploadedfile.InMemoryUploadedFile(
io.BytesIO(request.body),
'upload',
request.GET['file_name'],
content_type=request.GET['content_type'],
size=len(request.body),
charset=None)
response = djfw_views.save_uploaded_image(request, uploaded,
request.GET['file_name'])
return {
'id': response['id'],
'file_name': response['file_name'],
'url': response['image_link'],
'thumb': response['thumb_link'],
}
def get_form_kwargs(self):
kw = super().get_form_kwargs()
if self.request.method in ('POST', 'PUT'):
plain = kw['files']['encrypted']
raw = plain.read()
password = '******'
# ENCRYPTING!!!
encrypted = crypto.encrypt(raw, password)
stream = io.BytesIO(encrypted)
up = uploadedfile.InMemoryUploadedFile(stream, 'encrypted',
plain.name,
plain.content_type,
len(encrypted),
plain.charset)
kw['files']['encrypted'] = up
return kw
def put(self, request):
if not request.user.is_superuser:
raise exceptions.PermissionDenied()
file_name = request.GET['file_name']
content_type = request.GET['content_type']
uploaded = uploadedfile.InMemoryUploadedFile(io.BytesIO(request.body),
'upload',
file_name,
content_type=content_type,
size=len(request.body),
charset=None)
uploaded_file = models.UploadedFile(user=request.user,
filename=file_name,
file_size=len(request.body),
mime=content_type)
uploaded_file.save()
uploaded_file.body.save(f'{uploaded_file.pk}_{file_name}', uploaded)
uploaded_file.save()
return self.file_to_json(uploaded_file)
def reduce_image_size(self, image, size_limit=850000):
image_size = image.size[0] * image.size[1]
print('image size: {}'.format(image_size))
if image_size <= size_limit:
return self.image
else:
reduce_by = math.sqrt(image_size / size_limit)
size = int(image.width / reduce_by), int(image.height / reduce_by)
image = image.resize(size, Image.ANTIALIAS)
image_io = io.BytesIO()
image.save(image_io, format='JPEG')
image_file = uploadedfile.InMemoryUploadedFile(
file=image_io,
field_name=None,
name=os.path.split(self.image.name)[-1],
content_type='image/jpeg',
size=image_io.getbuffer().nbytes,
charset=None)
return image_file
def download_resource_file(url: str, target_name: str) -> Path:
"""Download a resource file and store it using GeoNode's `storage_manager`.
Downloads use the django `UploadedFile` helper classes. Depending on the size of the
remote resource, we may download it into an in-memory buffer or store it on a
temporary location on disk. After having downloaded the file, we use `storage_manager`
to save it in the appropriate location.
"""
response = requests.get(url, stream=True)
response.raise_for_status()
file_size = response.headers.get("Content-Length")
content_type = response.headers.get("Content-Type")
charset = response.apparent_encoding
size_threshold = config.get_setting(
"HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE")
if file_size is not None and int(file_size) < size_threshold:
logger.debug("Downloading to an in-memory buffer...")
buf = io.BytesIO()
file_ = uploadedfile.InMemoryUploadedFile(buf, None, target_name,
content_type, file_size,
charset)
else:
logger.debug("Downloading to a temporary file...")
file_ = uploadedfile.TemporaryUploadedFile(target_name, content_type,
file_size, charset)
# NOTE: there is no need to explicitly delete the file represented by
# `file_`, it is being deleted implicitly
with file_.open("wb+") as fd:
for chunk in response.iter_content(chunk_size=None,
decode_unicode=False):
fd.write(chunk)
fd.seek(0)
if storage_manager.exists(target_name):
logger.debug(f"file {target_name!r} already exists, replacing...")
storage_manager.delete(target_name)
file_name = storage_manager.save(target_name, fd)
result = Path(storage_manager.path(file_name))
return result
