def download_resource_file(url: str, target_name: str) -> str:
"""Download a resource file and store it using GeoNode's `storage_manager`.
Downloads use the django `UploadedFile` helper classes. Depending on the size of the
remote resource, we may download it into an in-memory buffer or store it on a
temporary location on disk. After having downloaded the file, we use `storage_manager`
to save it in the appropriate location.
"""
response = requests.get(url, stream=True)
response.raise_for_status()
file_size = response.headers.get("Content-Length")
content_type = response.headers.get("Content-Type")
charset = response.apparent_encoding
if file_size is not None and int(
file_size) < config.HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE:
logger.debug("Downloading to an in-memory buffer...")
file_ = uploadedfile.InMemoryUploadedFile(None, None, target_name,
content_type, file_size,
charset)
else:
logger.debug("Downloading to a temporary file...")
file_ = uploadedfile.TemporaryUploadedFile(target_name, content_type,
file_size, charset)
with file_.open("wb+") as fd:
for chunk in response.iter_content(chunk_size=None,
decode_unicode=False):
fd.write(chunk)
fd.seek(0)
result = storage_manager.save(target_name, fd)
return result
def inner(filename, content):
fp = uploadedfile.TemporaryUploadedFile(
name=filename + ".tempfile",
content_type='text/plain',
size=0,
charset='utf8',
)
fp.write(content)
fp.flush()
return fp
def _make_tempfile(self, filename, content):
fileobj = uploadedfile.TemporaryUploadedFile(
name=filename + ".tempfile",
content_type='text/plain',
size=0,
charset='utf8',
)
fileobj.write(content)
fileobj.flush()
return fileobj
def download_resource_file(url: str, target_name: str) -> Path:
"""Download a resource file and store it using GeoNode's `storage_manager`.
Downloads use the django `UploadedFile` helper classes. Depending on the size of the
remote resource, we may download it into an in-memory buffer or store it on a
temporary location on disk. After having downloaded the file, we use `storage_manager`
to save it in the appropriate location.
"""
response = requests.get(url, stream=True)
response.raise_for_status()
file_size = response.headers.get("Content-Length")
content_type = response.headers.get("Content-Type")
charset = response.apparent_encoding
size_threshold = config.get_setting(
"HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE")
if file_size is not None and int(file_size) < size_threshold:
logger.debug("Downloading to an in-memory buffer...")
buf = io.BytesIO()
file_ = uploadedfile.InMemoryUploadedFile(buf, None, target_name,
content_type, file_size,
charset)
else:
logger.debug("Downloading to a temporary file...")
file_ = uploadedfile.TemporaryUploadedFile(target_name, content_type,
file_size, charset)
# NOTE: there is no need to explicitly delete the file represented by
# `file_`, it is being deleted implicitly
with file_.open("wb+") as fd:
for chunk in response.iter_content(chunk_size=None,
decode_unicode=False):
fd.write(chunk)
fd.seek(0)
if storage_manager.exists(target_name):
logger.debug(f"file {target_name!r} already exists, replacing...")
storage_manager.delete(target_name)
file_name = storage_manager.save(target_name, fd)
result = Path(storage_manager.path(file_name))
return result