def get_download_response(request, docid, attachment=False):
"""
Returns a download response if user has access to download the document of a given id,
and an http response if they have no permissions to download it.
"""
document = get_object_or_404(Document, pk=docid)
if not request.user.has_perm('base.download_resourcebase',
obj=document.get_self_resource()):
return HttpResponse(loader.render_to_string(
'401.html',
context={
'error_message':
_("You are not allowed to view this document.")
},
request=request),
status=401)
if attachment:
register_event(request, EventType.EVENT_DOWNLOAD, document)
filename = slugify(os.path.splitext(os.path.basename(document.title))[0])
if document.files and storage_manager.exists(document.files[0]):
return DownloadResponse(storage_manager.open(document.files[0]).file,
basename=f'{filename}.{document.extension}',
attachment=attachment)
return HttpResponse("File is not available", status=404)
def update_thumbnail_urls_and_delete_curated_thumbs_folder(
apps, schema_editor):
query = '''
SELECT base_resourcebase.id, base_curatedthumbnail.img FROM base_resourcebase
INNER JOIN base_curatedthumbnail
ON base_resourcebase.id=base_curatedthumbnail.resource_id;
'''
c = connection.cursor()
c.execute(query)
results = c.fetchall()
for result in results:
resource_id, image = result
resource = ResourceBase.objects.filter(id=resource_id).first()
try:
bytes_file = storage_manager.open(image).read()
except Exception:
bytes_file = None
if resource and bytes_file:
filename = _generate_thumbnail_name(resource.get_real_instance())
resource.save_thumbnail(filename, bytes_file)
def create_document_thumbnail(self, object_id):
"""
Create thumbnail for a document.
"""
logger.debug(f"Generating thumbnail for document #{object_id}.")
try:
document = Document.objects.get(id=object_id)
except Document.DoesNotExist:
logger.error(f"Document #{object_id} does not exist.")
raise
image_path = None
image_file = None
if document.is_image:
dname = storage_manager.path(document.files[0])
if storage_manager.exists(dname):
image_file = storage_manager.open(dname, 'rb')
elif document.is_video or document.is_audio:
image_file = open(document.find_placeholder(), 'rb')
elif document.is_file:
dname = storage_manager.path(document.files[0])
try:
document_location = storage_manager.path(dname)
except NotImplementedError as e:
logger.debug(e)
document_location = storage_manager.url(dname)
try:
image_path = render_document(document_location)
if image_path is not None:
try:
image_file = open(image_path, 'rb')
except Exception as e:
logger.debug(
f"Failed to render document #{object_id}: {e}")
else:
logger.debug(f"Failed to render document #{object_id}")
except ConversionError as e:
logger.debug(f"Could not convert document #{object_id}: {e}.")
except NotImplementedError as e:
logger.debug(f"Failed to render document #{object_id}: {e}")
thumbnail_content = None
try:
try:
thumbnail_content = generate_thumbnail_content(image_file)
except Exception as e:
logger.debug(
f"Could not generate thumbnail, falling back to 'placeholder': {e}"
)
thumbnail_content = generate_thumbnail_content(
document.find_placeholder())
except Exception as e:
logger.error(f"Could not generate thumbnail: {e}")
return
finally:
if image_file is not None:
image_file.close()
if image_path is not None:
os.remove(image_path)
if not thumbnail_content:
logger.warning(f"Thumbnail for document #{object_id} empty.")
filename = f'document-{document.uuid}-thumb.png'
document.save_thumbnail(filename, thumbnail_content)
logger.debug(f"Thumbnail for document #{object_id} created.")
def thumb_open(filename):
"""Returns file handler of a thumbnail on the storage"""
return storage_manager.open(thumb_path(filename))
def download(request, resourceid, sender=Dataset):
_not_authorized = _("You are not authorized to download this resource.")
_not_permitted = _("You are not permitted to save or edit this resource.")
_no_files_found = _(
"No files have been found for this resource. Please, contact a system administrator."
)
instance = resolve_object(request,
sender, {'pk': resourceid},
permission='base.download_resourcebase',
permission_msg=_not_permitted)
if isinstance(instance, ResourceBase):
dataset_files = []
file_list = [] # Store file info to be returned
try:
files = instance.resourcebase_ptr.files
# Copy all Dataset related files into a temporary folder
for file_path in files:
if storage_manager.exists(file_path):
dataset_files.append(file_path)
filename = os.path.basename(file_path)
file_list.append({
"name":
filename,
"data_iter":
storage_manager.open(file_path),
})
else:
return HttpResponse(loader.render_to_string(
'401.html',
context={
'error_title': _("No files found."),
'error_message': _no_files_found
},
request=request),
status=404)
# Check we can access the original files
if not dataset_files:
return HttpResponse(loader.render_to_string(
'401.html',
context={
'error_title': _("No files found."),
'error_message': _no_files_found
},
request=request),
status=404)
# ZIP everything and return
target_file_name = "".join([instance.name, ".zip"])
target_zip = zipstream.ZipFile(mode='w',
compression=zipstream.ZIP_DEFLATED,
allowZip64=True)
# Iterable: Needed when the file_info has it's data as a stream
def _iterable(source_iter):
while True:
buf = source_iter.read(BUFFER_CHUNK_SIZE)
if not buf:
break
yield buf
# Add files to zip
for file_info in file_list:
target_zip.write_iter(arcname=file_info['name'],
iterable=_iterable(
file_info['data_iter']))
register_event(request, 'download', instance)
# Streaming content response
response = StreamingHttpResponse(target_zip,
content_type='application/zip')
response[
'Content-Disposition'] = f'attachment; filename="{target_file_name}"'
return response
except (NotImplementedError, Upload.DoesNotExist):
traceback.print_exc()
tb = traceback.format_exc()
logger.debug(tb)
return HttpResponse(loader.render_to_string(
'401.html',
context={
'error_title': _("No files found."),
'error_message': _no_files_found
},
request=request),
status=404)
return HttpResponse(loader.render_to_string('401.html',
context={
'error_title':
_("Not Authorized"),
'error_message':
_not_authorized
},
request=request),
status=403)
def _execute_resource_import(self, instance, files: list, user, action_type: str, importer_session_opts: typing.Optional[typing.Dict] = None):
from geonode.upload.files import ALLOWED_EXTENSIONS
session_opts = dict(importer_session_opts) if importer_session_opts is not None else {}
spatial_files_type = get_spatial_files_dataset_type(ALLOWED_EXTENSIONS, files)
if not spatial_files_type:
raise Exception(f"No suitable Spatial Files avaialable for 'ALLOWED_EXTENSIONS' = {ALLOWED_EXTENSIONS}.")
upload_session, _ = Upload.objects.get_or_create(resource=instance.get_real_instance().resourcebase_ptr, user=user)
upload_session.resource = instance.get_real_instance().resourcebase_ptr
upload_session.save()
_name = instance.get_real_instance().name
if not _name:
_name = session_opts.get('name', None) or os.path.splitext(os.path.basename(spatial_files_type.base_file))[0]
instance.get_real_instance().name = _name
gs_dataset = None
try:
gs_dataset = gs_catalog.get_layer(_name)
except Exception as e:
logger.debug(e)
_workspace = None
_target_store = None
if gs_dataset:
_target_store = gs_dataset.resource.store.name if instance.get_real_instance().subtype == 'vector' else None
_workspace = gs_dataset.resource.workspace.name if gs_dataset.resource.workspace else None
if not _workspace:
_workspace = session_opts.get('workspace', instance.get_real_instance().workspace)
if not _workspace:
_workspace = instance.get_real_instance().workspace or settings.DEFAULT_WORKSPACE
if not _target_store:
if instance.get_real_instance().subtype == 'vector' or spatial_files_type.dataset_type == 'vector':
_dsname = ogc_server_settings.datastore_db['NAME']
_ds = create_geoserver_db_featurestore(store_name=_dsname, workspace=_workspace)
if _ds:
_target_store = session_opts.get('target_store', None) or _dsname
# opening Import session for the selected layer
# Let's reset the connections first
gs_catalog._cache.clear()
gs_catalog.reset()
# Let's now try the new ingestion
import_session = gs_uploader.start_import(
import_id=upload_session.id,
name=_name,
target_store=_target_store
)
upload_session.set_processing_state(enumerations.STATE_PROCESSED)
upload_session.import_id = import_session.id
upload_session.name = _name
upload_session.complete = True
upload_session.processed = True
upload_session.save()
_gs_import_session_info = GeoServerImporterSessionInfo(
upload_session=upload_session,
import_session=import_session,
spatial_files_type=spatial_files_type,
dataset_name=None,
workspace=_workspace,
target_store=_target_store
)
_local_files = []
_temporary_files = []
try:
for _f in files:
if os.path.exists(_f) and os.path.isfile(_f):
_local_files.append(os.path.abspath(_f))
try:
os.close(_f)
except Exception:
pass
else:
_suffix = os.path.splitext(os.path.basename(_f))[1] if len(os.path.splitext(os.path.basename(_f))) else None
with tempfile.NamedTemporaryFile(mode="wb+", delete=False, dir=settings.MEDIA_ROOT, suffix=_suffix) as _tmp_file:
_tmp_file.write(storage_manager.open(_f, 'rb+').read())
_tmp_file.seek(0)
_tmp_file_name = f'{_tmp_file.name}'
_local_files.append(os.path.abspath(_tmp_file_name))
_temporary_files.append(os.path.abspath(_tmp_file_name))
try:
storage_manager.close(_f)
except Exception:
pass
except Exception as e:
logger.exception(e)
if _local_files:
try:
import_session.upload_task(_local_files)
task = import_session.tasks[0]
# Changing layer name, mode and target
task.layer.set_target_layer_name(_name)
task.set_update_mode(action_type.upper())
task.set_target(
store_name=_target_store,
workspace=_workspace
)
transforms = session_opts.get('transforms', None)
if transforms:
task.set_transforms(transforms)
# Starting import process
import_session.commit()
import_session = import_session.reload()
try:
# Updating Resource with the files replaced
if action_type.lower() == 'replace':
updated_files_list = storage_manager.replace(instance, files)
# Using update instead of save in order to avoid calling
# side-effect function of the resource
r = ResourceBase.objects.filter(id=instance.id)
r.update(**updated_files_list)
else:
instance.files = files
except Exception as e:
logger.exception(e)
_gs_import_session_info.import_session = import_session
_gs_import_session_info.dataset_name = import_session.tasks[0].layer.name
finally:
for _f in _temporary_files:
try:
os.remove(_f)
except Exception as e:
logger.debug(e)
return _gs_import_session_info
def download(request, resourceid, sender=Dataset):
_not_authorized = _("You are not authorized to download this resource.")
_not_permitted = _("You are not permitted to save or edit this resource.")
_no_files_found = _("No files have been found for this resource. Please, contact a system administrator.")
instance = resolve_object(request,
sender,
{'pk': resourceid},
permission='base.download_resourcebase',
permission_msg=_not_permitted)
if isinstance(instance, ResourceBase):
# Create Target Folder
dirpath = tempfile.mkdtemp(dir=settings.STATIC_ROOT)
dir_time_suffix = get_dir_time_suffix()
target_folder = os.path.join(dirpath, dir_time_suffix)
if not os.path.exists(target_folder):
os.makedirs(target_folder)
dataset_files = []
try:
files = instance.resourcebase_ptr.files
# Copy all Dataset related files into a temporary folder
for file_path in files:
if storage_manager.exists(file_path):
dataset_files.append(file_path)
filename = os.path.basename(file_path)
with open(f"{target_folder}/{filename}", 'wb+') as f:
f.write(storage_manager.open(file_path).read())
else:
return HttpResponse(
loader.render_to_string(
'401.html',
context={
'error_title': _("No files found."),
'error_message': _no_files_found
},
request=request), status=404)
# Check we can access the original files
if not dataset_files:
return HttpResponse(
loader.render_to_string(
'401.html',
context={
'error_title': _("No files found."),
'error_message': _no_files_found
},
request=request), status=404)
# ZIP everything and return
target_file_name = "".join([instance.name, ".zip"])
target_file = os.path.join(dirpath, target_file_name)
zip_dir(target_folder, target_file)
register_event(request, 'download', instance)
response = HttpResponse(
content=open(target_file, mode='rb'),
status=200,
content_type="application/zip")
response['Content-Disposition'] = f'attachment; filename="{target_file_name}"'
return response
except (NotImplementedError, Upload.DoesNotExist):
traceback.print_exc()
tb = traceback.format_exc()
logger.debug(tb)
return HttpResponse(
loader.render_to_string(
'401.html',
context={
'error_title': _("No files found."),
'error_message': _no_files_found
},
request=request), status=404)
finally:
if target_folder is not None:
shutil.rmtree(target_folder, ignore_errors=True)
return HttpResponse(
loader.render_to_string(
'401.html',
context={
'error_title': _("Not Authorized"),
'error_message': _not_authorized
},
request=request), status=403)
