def delete_orphaned_datasets():
"""Delete orphaned layer files."""
deleted = []
_, files = storage_manager.listdir("layers")
for filename in files:
if Dataset.objects.filter(file__icontains=filename).count() == 0:
logger.debug(f"Deleting orphaned dataset file {filename}")
try:
storage_manager.delete(os.path.join("layers", filename))
deleted.append(filename)
except NotImplementedError as e:
logger.error(
f"Failed to delete orphaned dataset file '{filename}': {e}")
return deleted
def delete_orphaned_document_files():
"""
Deletes orphaned files of deleted documents.
"""
deleted = []
_, files = storage_manager.listdir(os.path.join("documents", "document"))
for filename in files:
if Document.objects.filter(doc_file__contains=filename).count() == 0:
logger.debug(f"Deleting orphaned document {filename}")
try:
storage_manager.delete(os.path.join(
os.path.join("documents", "document"), filename))
deleted.append(filename)
except NotImplementedError as e:
logger.error(
f"Failed to delete orphaned document '{filename}': {e}")
return deleted
def delete(self, *args, **kwargs):
importer_locations = []
super().delete(*args, **kwargs)
try:
session = gs_uploader.get_session(self.import_id)
except (NotFound, Exception):
session = None
if session:
for task in session.tasks:
if getattr(task, 'data'):
importer_locations.append(
getattr(task.data, 'location'))
try:
session.delete()
except Exception:
logging.warning('error deleting upload session')
# we delete directly the folder with the files of the resource
if self.resource:
for _file in self.resource.files:
try:
if storage_manager.exists(_file):
storage_manager.delete(_file)
except Exception as e:
logger.warning(e)
# Do we want to delete the files also from the resource?
ResourceBase.objects.filter(id=self.resource.id).update(files={})
for _location in importer_locations:
try:
shutil.rmtree(_location)
except Exception as e:
logger.warning(e)
# here we are deleting the local that soon will be removed
if self.upload_dir and os.path.exists(self.upload_dir):
try:
shutil.rmtree(self.upload_dir)
except Exception as e:
logger.warning(e)
def download_resource_file(url: str, target_name: str) -> Path:
"""Download a resource file and store it using GeoNode's `storage_manager`.
Downloads use the django `UploadedFile` helper classes. Depending on the size of the
remote resource, we may download it into an in-memory buffer or store it on a
temporary location on disk. After having downloaded the file, we use `storage_manager`
to save it in the appropriate location.
"""
response = requests.get(url, stream=True)
response.raise_for_status()
file_size = response.headers.get("Content-Length")
content_type = response.headers.get("Content-Type")
charset = response.apparent_encoding
size_threshold = config.get_setting(
"HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE")
if file_size is not None and int(file_size) < size_threshold:
logger.debug("Downloading to an in-memory buffer...")
buf = io.BytesIO()
file_ = uploadedfile.InMemoryUploadedFile(buf, None, target_name,
content_type, file_size,
charset)
else:
logger.debug("Downloading to a temporary file...")
file_ = uploadedfile.TemporaryUploadedFile(target_name, content_type,
file_size, charset)
# NOTE: there is no need to explicitly delete the file represented by
# `file_`, it is being deleted implicitly
with file_.open("wb+") as fd:
for chunk in response.iter_content(chunk_size=None,
decode_unicode=False):
fd.write(chunk)
fd.seek(0)
if storage_manager.exists(target_name):
logger.debug(f"file {target_name!r} already exists, replacing...")
storage_manager.delete(target_name)
file_name = storage_manager.save(target_name, fd)
result = Path(storage_manager.path(file_name))
return result
def remove_thumb(filename):
"""Delete a thumbnail from storage"""
path = thumb_path(filename)
if storage_manager.exists(path):
storage_manager.delete(path)
def remove_thumb(filename):
"""Delete a thumbnail from storage"""
storage_manager.delete(thumb_path(filename))