def geoserver_create_thumbnail(self,
instance_id,
overwrite=True,
check_bbox=True):
"""
Runs create_gs_thumbnail.
"""
instance = None
try:
instance = ResourceBase.objects.get(id=instance_id).get_real_instance()
except Exception:
logger.error(f"Resource id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
try:
create_gs_thumbnail(instance,
overwrite=overwrite,
check_bbox=check_bbox)
logger.debug(
f"... Created Thumbnail for Layer {instance.title}")
except Exception as e:
geoserver_create_thumbnail.retry(exc=e)
def geoserver_set_style(
self,
instance_id,
base_file):
"""
Sets styles from SLD file.
"""
instance = None
try:
instance = Dataset.objects.get(id=instance_id)
except Dataset.DoesNotExist:
logger.debug(f"Dataset id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
try:
sld = open(base_file, "rb").read()
set_dataset_style(
instance,
instance.alternate,
sld,
base_file=base_file)
except Exception as e:
logger.exception(e)
def geoserver_create_thumbnail(self,
instance_id,
overwrite=True,
check_bbox=True):
"""
Runs create_gs_thumbnail.
"""
instance = None
try:
instance = ResourceBase.objects.get(id=instance_id).get_real_instance()
except Exception:
logger.error(f"Resource id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
instance.set_processing_state(enumerations.STATE_RUNNING)
try:
instance.set_dirty_state()
create_gs_thumbnail(instance,
overwrite=overwrite,
check_bbox=check_bbox)
logger.debug(
f"... Created Thumbnail for Dataset {instance.title}")
except Exception as e:
geoserver_create_thumbnail.retry(exc=e)
finally:
instance.set_processing_state(enumerations.STATE_PROCESSED)
def _update_upload_session_state(self, upload_session_id: int):
"""Task invoked by 'upload_workflow.chord' in order to process all the 'PENDING' Upload tasks."""
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
_upload = Upload.objects.get(id=upload_session_id)
session = _upload.get_session.import_session
if not session or session.state != enumerations.STATE_COMPLETE:
session = gs_uploader.get_session(_upload.import_id)
if session:
try:
content = next_step_response(None, _upload.get_session).content
if isinstance(content, bytes):
content = content.decode('UTF-8')
response_json = json.loads(content)
_success = response_json.get('success', False)
_redirect_to = response_json.get('redirect_to', '')
if _success:
if 'upload/final' not in _redirect_to and 'upload/check' not in _redirect_to:
_upload.set_resume_url(_redirect_to)
_upload.set_processing_state(enumerations.STATE_WAITING)
else:
if session.state == enumerations.STATE_COMPLETE and _upload.state == enumerations.STATE_PENDING:
if not _upload.resource or not _upload.resource.processed:
final_step_view(None, _upload.get_session)
_upload.set_processing_state(enumerations.STATE_RUNNING)
except (NotFound, Exception) as e:
logger.exception(e)
if _upload.state not in (enumerations.STATE_COMPLETE, enumerations.STATE_PROCESSED):
_upload.set_processing_state(enumerations.STATE_INVALID)
if _upload.resource:
resource_manager.delete(_upload.resource.uuid)
def geoserver_cascading_delete(self, *args, **kwargs):
"""
Runs cascading_delete.
"""
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
return cascading_delete(*args, **kwargs)
def geoserver_update_datasets(self, *args, **kwargs):
"""
Runs update layers.
"""
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
return gs_slurp(*args, **kwargs)
def geoserver_create_style(
self,
instance_id,
name,
sld_file,
tempdir):
"""
Sets or create styles from Upload Session.
"""
instance = None
try:
instance = Dataset.objects.get(id=instance_id)
except Dataset.DoesNotExist:
logger.debug(f"Dataset id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True and instance:
if sld_file and os.path.exists(sld_file) and os.access(sld_file, os.R_OK):
f = None
if os.path.isfile(sld_file):
try:
f = open(sld_file)
except Exception:
pass
elif tempdir and os.path.exists(tempdir):
if os.path.isfile(os.path.join(tempdir, sld_file)):
try:
f = open(os.path.join(tempdir, sld_file))
except Exception:
pass
if f:
sld = f.read()
f.close()
if not gs_catalog.get_style(name=name, workspace=settings.DEFAULT_WORKSPACE):
style = gs_catalog.create_style(
name,
sld,
raw=True,
workspace=settings.DEFAULT_WORKSPACE)
gs_dataset = gs_catalog.get_layer(name)
_default_style = gs_dataset.default_style
gs_dataset.default_style = style
gs_catalog.save(gs_dataset)
set_styles(instance, gs_catalog)
try:
gs_catalog.delete(_default_style)
Link.objects.filter(
resource=instance.resourcebase_ptr,
name='Legend',
url__contains=f'STYLE={_default_style.name}').delete()
except Exception as e:
logger.exception(e)
else:
get_sld_for(gs_catalog, instance)
else:
get_sld_for(gs_catalog, instance)
def get_resume_url(self):
if self.state == Upload.STATE_WAITING and self.import_id:
return f"{reverse('data_upload')}?id={self.import_id}"
else:
session = None
try:
if not self.import_id:
raise NotFound
session = self.get_session.import_session
if not session or session.state != Upload.STATE_COMPLETE:
session = gs_uploader.get_session(self.import_id)
except (NotFound, Exception):
if self.state not in (Upload.STATE_COMPLETE,
Upload.STATE_PROCESSED):
self.state = Upload.STATE_INVALID
Upload.objects.filter(id=self.id).update(
state=Upload.STATE_INVALID)
if session:
lock_id = f'{self.import_id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
try:
content = next_step_response(
None, self.get_session).content
if isinstance(content, bytes):
content = content.decode('UTF-8')
response_json = json.loads(content)
if response_json[
'success'] and 'redirect_to' in response_json:
if 'upload/final' not in response_json[
'redirect_to'] and 'upload/check' not in response_json[
'redirect_to']:
self.state = Upload.STATE_WAITING
Upload.objects.filter(id=self.id).update(
state=Upload.STATE_WAITING)
return f"{reverse('data_upload')}?id={self.import_id}"
else:
next = get_next_step(self.get_session)
if next == 'final' and session.state == Upload.STATE_COMPLETE and self.state == Upload.STATE_PENDING:
if not self.layer or not self.layer.processed:
from .views import final_step_view
final_step_view(
None, self.get_session)
self.state = Upload.STATE_RUNNING
Upload.objects.filter(
id=self.id).update(
state=Upload.STATE_RUNNING)
except (NotFound, Exception) as e:
logger.exception(e)
if self.state not in (Upload.STATE_COMPLETE,
Upload.STATE_PROCESSED):
self.state = Upload.STATE_INVALID
Upload.objects.filter(id=self.id).update(
state=Upload.STATE_INVALID)
return None
def geoserver_post_save_datasets(self, instance_id, *args, **kwargs):
"""
Runs update layers.
"""
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
sync_instance_with_geoserver(instance_id, *args, **kwargs)
# Updating HAYSTACK Indexes if needed
if settings.HAYSTACK_SEARCH:
call_command('update_index')
def _upload_session_cleanup(self, upload_session_id: int):
"""Task invoked by 'upload_workflow.chord' in order to remove and cleanup all the 'INVALID' stale Upload tasks."""
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
try:
_upload = Upload.objects.get(id=upload_session_id)
if _upload.resource:
resource_manager.delete(_upload.resource.uuid)
_upload.delete()
logger.debug(f"Upload {upload_session_id} deleted with state {_upload.state}.")
except Exception as e:
logger.error(f"Upload {upload_session_id} errored with exception {e}.")
def probe_services(self):
# The cache key consists of the task name and the MD5 digest
# of the name.
name = b'probe_services'
hexdigest = md5(name).hexdigest()
lock_id = f'{name.decode()}-lock-{hexdigest}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
for service in models.Service.objects.all():
try:
service.probe = service.probe_service()
service.save()
except Exception as e:
logger.error(e)
def geoserver_delete_map(self, object_id):
"""
Deletes a map and the associated map layers.
"""
from geonode.maps.models import Map
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
try:
map_obj = Map.objects.get(id=object_id)
except Map.DoesNotExist:
return
map_obj.dataset_set.all().delete()
map_obj.delete()
def geoserver_finalize_upload(self, import_id, instance_id, permissions,
created, xml_file, sld_file, sld_uploaded,
tempdir):
"""
Finalize Layer and GeoServer configuration:
- Sets Layer Metadata from XML and updates GeoServer Layer accordingly.
- Sets Default Permissions
"""
instance = None
try:
instance = Layer.objects.get(id=instance_id)
except Layer.DoesNotExist:
logger.debug(f"Layer id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
from geonode.upload.models import Upload
upload = Upload.objects.get(import_id=import_id)
upload.layer = instance
upload.save()
# Sanity checks
if isinstance(xml_file, list):
if len(xml_file) > 0:
xml_file = xml_file[0]
else:
xml_file = None
elif not isinstance(xml_file, six.string_types):
xml_file = None
if xml_file and os.path.exists(xml_file) and os.access(
xml_file, os.R_OK):
instance.metadata_uploaded = True
identifier, vals, regions, keywords = set_metadata(
open(xml_file).read())
try:
gs_resource = gs_catalog.get_resource(
name=instance.name,
store=instance.store,
workspace=instance.workspace)
except Exception:
try:
gs_resource = gs_catalog.get_resource(
name=instance.alternate,
store=instance.store,
workspace=instance.workspace)
except Exception:
try:
gs_resource = gs_catalog.get_resource(
name=instance.alternate or instance.typename)
except Exception:
gs_resource = None
if vals:
title = vals.get('title', '')
abstract = vals.get('abstract', '')
# Updating GeoServer resource
gs_resource.title = title
gs_resource.abstract = abstract
gs_catalog.save(gs_resource)
else:
vals = {}
vals.update(
dict(uuid=instance.uuid,
name=instance.name,
owner=instance.owner,
store=gs_resource.store.name,
storeType=gs_resource.store.resource_type,
alternate=gs_resource.store.workspace.name + ':' +
gs_resource.name,
title=gs_resource.title or gs_resource.store.name,
abstract=gs_resource.abstract or ''))
instance.metadata_xml = xml_file
regions_resolved, regions_unresolved = resolve_regions(regions)
keywords.extend(regions_unresolved)
# Assign the regions (needs to be done after saving)
regions_resolved = list(set(regions_resolved))
if regions_resolved:
if len(regions_resolved) > 0:
if not instance.regions:
instance.regions = regions_resolved
else:
instance.regions.clear()
instance.regions.add(*regions_resolved)
# Assign the keywords (needs to be done after saving)
keywords = list(set(keywords))
if keywords:
if len(keywords) > 0:
if not instance.keywords:
instance.keywords = keywords
else:
instance.keywords.add(*keywords)
# set model properties
defaults = {}
for key, value in vals.items():
if key == 'spatial_representation_type':
value = SpatialRepresentationType(identifier=value)
elif key == 'topic_category':
value, created = TopicCategory.objects.get_or_create(
identifier=value.lower(),
defaults={
'description': '',
'gn_description': value
})
key = 'category'
defaults[key] = value
else:
defaults[key] = value
# Save all the modified information in the instance without triggering signals.
try:
if not defaults.get('title', title):
defaults['title'] = instance.title or instance.name
if not defaults.get('abstract', abstract):
defaults['abstract'] = instance.abstract or ''
to_update = {}
to_update['charset'] = defaults.pop(
'charset', instance.charset)
to_update['storeType'] = defaults.pop(
'storeType', instance.storeType)
for _key in ('name', 'workspace', 'store', 'storeType',
'alternate', 'typename'):
if _key in defaults:
to_update[_key] = defaults.pop(_key)
else:
to_update[_key] = getattr(instance, _key)
to_update.update(defaults)
with transaction.atomic():
ResourceBase.objects.filter(
id=instance.resourcebase_ptr.id).update(**defaults)
Layer.objects.filter(id=instance.id).update(
**to_update)
# Refresh from DB
instance.refresh_from_db()
except IntegrityError:
raise
if sld_uploaded:
geoserver_set_style(instance.id, sld_file)
else:
geoserver_create_style(instance.id, instance.name, sld_file,
tempdir)
logger.debug(
'Finalizing (permissions and notifications) Layer {0}'.format(
instance))
instance.handle_moderated_uploads()
if permissions is not None:
logger.debug(
f'Setting permissions {permissions} for {instance.name}')
instance.set_permissions(permissions, created=created)
elif created:
logger.debug(
f'Setting default permissions for {instance.name}')
instance.set_default_permissions()
try:
# Update the upload sessions
geonode_upload_sessions = UploadSession.objects.filter(
resource=instance)
geonode_upload_sessions.update(processed=False)
instance.upload_session = geonode_upload_sessions.first()
except Exception as e:
logger.exception(e)
instance.save(notify=not created)
try:
logger.debug(
f"... Cleaning up the temporary folders {tempdir}")
if tempdir and os.path.exists(tempdir):
shutil.rmtree(tempdir)
finally:
upload.complete = True
upload.save()
signals.upload_complete.send(sender=geoserver_finalize_upload,
layer=instance)
def collect_metric(**options):
# Exit early if MONITORING_ENABLED=False
if not settings.MONITORING_ENABLED:
return
# Avoid possible module circular dependency issues
from geonode.monitoring.models import Service
from geonode.monitoring.collector import CollectorAPI
_start_time = None
_end_time = None
# The cache key consists of the task name and the MD5 digest
# of the name.
name = b'collect_metric'
hexdigest = md5(name).hexdigest()
lock_id = f'{name.decode()}-lock-{hexdigest}'
_start_time = _end_time = datetime.utcnow().isoformat()
log.info(f'[{lock_id}] Collecting Metrics - started @ {_start_time}')
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
log.info(
f'[{lock_id}] Collecting Metrics - [...acquired lock] @ {_start_time}'
)
try:
oservice = options['service']
if not oservice:
services = Service.objects.all()
else:
services = [oservice]
if options['list_services']:
print('available services')
for s in services:
print(' ', s.name, '(', s.url, ')')
print(' type', s.service_type.name)
print(' running on', s.host.name, s.host.ip)
print(' active:', s.active)
if s.last_check:
print(' last check:', s.last_check)
else:
print(' not checked yet')
print(' ')
return
c = CollectorAPI()
for s in services:
try:
run_check(s,
collector=c,
since=options['since'],
until=options['until'],
force_check=options['force_check'],
format=options['format'])
except Exception as e:
log.warning(e)
if not options['do_not_clear']:
log.info("Clearing old data")
c.clear_old_data()
if options['emit_notifications']:
log.info("Processing notifications for %s",
options['until'])
# s = Service.objects.first()
# interval = s.check_interval
# now = datetime.utcnow().replace(tzinfo=pytz.utc)
# notifications_check = now - interval
c.emit_notifications() # notifications_check))
_end_time = datetime.utcnow().isoformat()
log.info(
f'[{lock_id}] Collecting Metrics - finished @ {_end_time}')
except Exception as e:
log.info(
f'[{lock_id}] Collecting Metrics - errored @ {_end_time}')
log.exception(e)
log.info(f'[{lock_id}] Collecting Metrics - exit @ {_end_time}')
return (_start_time, _end_time)
def resouce_service_dispatcher(self, execution_id: str):
"""Performs a Resource Service request asynchronously.
This is the main Resource Service API dispatcher.
The method looks for avaialable `ExecutionRequests` with status `READY` and triggers the
`func_name` method of the `resource_manager` with the `input_params`.
It finally updates the `status` of the request.
A client is able to query the `status_url` endpoint in order to get the current `status` other than
the `output_params`.
"""
with AcquireLock(execution_id) as lock:
if lock.acquire() is True:
_exec_request = ExecutionRequest.objects.filter(
exec_id=execution_id)
if _exec_request.exists():
_request = _exec_request.get()
if _request.status == ExecutionRequest.STATUS_READY:
_exec_request.update(
status=ExecutionRequest.STATUS_RUNNING)
_request.refresh_from_db()
if hasattr(resource_manager, _request.func_name):
try:
_signature = signature(
getattr(resource_manager, _request.func_name))
_args = []
_kwargs = {}
for _param_name in _signature.parameters:
if _request.input_params and _request.input_params.get(
_param_name, None):
_param = _signature.parameters.get(
_param_name)
_param_value = _get_param_value(
_param,
_request.input_params.get(_param_name))
if _param.kind == Parameter.POSITIONAL_ONLY:
_args.append(_param_value)
else:
_kwargs[_param_name] = _param_value
_bindings = _signature.bind(*_args, **_kwargs)
_bindings.apply_defaults()
_output = getattr(resource_manager,
_request.func_name)(
*_bindings.args,
**_bindings.kwargs)
_output_params = {}
if _output is not None and _signature.return_annotation != Signature.empty:
if _signature.return_annotation.__module__ == 'builtins':
_output_params = {"output": _output}
elif _signature.return_annotation == ResourceBase or isinstance(
_output, ResourceBase):
_output_params = {
"output": {
"uuid": _output.uuid
}
}
else:
_output_params = {"output": None}
_exec_request.update(
status=ExecutionRequest.STATUS_FINISHED,
finished=datetime.now(),
output_params=_output_params)
_request.refresh_from_db()
except Exception as e:
logger.exception(e)
_exec_request.update(
status=ExecutionRequest.STATUS_FAILED,
finished=datetime.now(),
output_params={
"error":
_(f"Error occurred while executin the operation: '{_request.func_name}'"
),
"exception":
str(e)
})
_request.refresh_from_db()
else:
_exec_request.update(
status=ExecutionRequest.STATUS_FAILED,
finished=datetime.now(),
output_params={
"error":
_(f"Could not find the operation name: '{_request.func_name}'"
)
})
_request.refresh_from_db()
logger.error(
f"WARNING: The requested ExecutionRequest with 'exec_id'={execution_id} was not found!"
)
def finalize_incomplete_session_uploads(self, *args, **kwargs):
"""The task periodically checks for pending and stale Upload sessions.
It runs every 600 seconds (see the PeriodTask on geonode.upload._init_),
checks first for expired stale Upload sessions and schedule them for cleanup.
We have to make sure To NOT Delete those Unprocessed Ones,
which are in live sessions.
After removing the stale ones, it collects all the unprocessed and runs them
in parallel."""
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
_upload_ids = []
_upload_tasks = []
# Check first if we need to delete stale sessions
expiry_time = now() - timedelta(hours=UPLOAD_SESSION_EXPIRY_HOURS)
for _upload in Upload.objects.exclude(
state=enumerations.STATE_PROCESSED).exclude(
date__gt=expiry_time):
_upload.set_processing_state(enumerations.STATE_INVALID)
_upload_ids.append(_upload.id)
_upload_tasks.append(
_upload_session_cleanup.signature(args=(_upload.id, )))
upload_workflow_finalizer = _upload_workflow_finalizer.signature(
args=(
'_upload_session_cleanup',
_upload_ids,
),
immutable=True).on_error(
_upload_workflow_error.signature(args=(
'_upload_session_cleanup',
_upload_ids,
),
immutable=True))
upload_workflow = chord(_upload_tasks,
body=upload_workflow_finalizer)
upload_workflow.apply_async()
# Let's finish the valid ones
_processing_states = (enumerations.STATE_RUNNING,
enumerations.STATE_INVALID,
enumerations.STATE_PROCESSED)
for _upload in Upload.objects.exclude(
state__in=_processing_states):
session = None
try:
if not _upload.import_id:
raise NotFound
session = _upload.get_session.import_session
if not session or session.state != enumerations.STATE_COMPLETE:
session = gs_uploader.get_session(_upload.import_id)
except (NotFound, Exception) as e:
logger.exception(e)
session = None
if _upload.state not in (enumerations.STATE_COMPLETE,
enumerations.STATE_PROCESSED):
_upload.set_processing_state(
enumerations.STATE_INVALID)
if _upload.resource:
resource_manager.delete(_upload.resource.uuid)
if session:
_upload_ids.append(_upload.id)
_upload_tasks.append(
_update_upload_session_state.signature(
args=(_upload.id, )))
upload_workflow_finalizer = _upload_workflow_finalizer.signature(
args=(
'_update_upload_session_state',
_upload_ids,
),
immutable=True).on_error(
_upload_workflow_error.signature(args=(
'_update_upload_session_state',
_upload_ids,
),
immutable=True))
upload_workflow = chord(_upload_tasks,
body=upload_workflow_finalizer)
upload_workflow.apply_async()
def geoserver_post_save_layers(self, instance_id, *args, **kwargs):
"""
Runs update layers.
"""
instance = None
try:
instance = Layer.objects.get(id=instance_id)
except Layer.DoesNotExist:
logger.debug(f"Layer id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
# Don't run this signal if is a Layer from a remote service
if getattr(instance, "remote_service", None) is not None:
return
if instance.storeType == "remoteStore":
return
# Don't run this signal handler if it is a tile layer or a remote store (Service)
# Currently only gpkg files containing tiles will have this type & will be served via MapProxy.
if hasattr(instance, 'storeType') and getattr(
instance, 'storeType') in ['tileStore', 'remoteStore']:
return instance
if isinstance(instance, ResourceBase):
if hasattr(instance, 'layer'):
instance = instance.layer
else:
return
geonode_upload_sessions = UploadSession.objects.filter(
resource=instance)
geonode_upload_sessions.update(processed=False)
gs_resource = None
values = None
_tries = 0
_max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 2)
# If the store in None then it's a new instance from an upload,
# only in this case run the geoserver_upload method
if not instance.store or getattr(instance, 'overwrite', False):
base_file, info = instance.get_base_file()
# There is no need to process it if there is no file.
if base_file is None:
return
gs_name, workspace, values, gs_resource = geoserver_upload(
instance,
base_file.file.path,
instance.owner,
instance.name,
overwrite=True,
title=instance.title,
abstract=instance.abstract,
charset=instance.charset)
values, gs_resource = fetch_gs_resource(instance, values, _tries)
while not gs_resource and _tries < _max_tries:
values, gs_resource = fetch_gs_resource(
instance, values, _tries)
_tries += 1
# Get metadata links
metadata_links = []
for link in instance.link_set.metadata():
metadata_links.append((link.mime, link.name, link.url))
if gs_resource:
logger.debug("Found geoserver resource for this layer: %s" %
instance.name)
gs_resource.metadata_links = metadata_links
instance.gs_resource = gs_resource
# Update Attribution link
if instance.poc:
# gsconfig now utilizes an attribution dictionary
gs_resource.attribution = {
'title': str(instance.poc),
'width': None,
'height': None,
'href': None,
'url': None,
'type': None
}
profile = get_user_model().objects.get(
username=instance.poc.username)
site_url = settings.SITEURL.rstrip(
'/') if settings.SITEURL.startswith(
'http') else settings.SITEURL
gs_resource.attribution_link = site_url + profile.get_absolute_url(
)
"""Get information from geoserver.
The attributes retrieved include:
* Bounding Box
* SRID
* Download links (WMS, WCS or WFS and KML)
* Styles (SLD)
"""
try:
# This is usually done in Layer.pre_save, however if the hooks
# are bypassed by custom create/updates we need to ensure the
# bbox is calculated properly.
bbox = gs_resource.native_bbox
instance.set_bbox_polygon(
[bbox[0], bbox[2], bbox[1], bbox[3]],
gs_resource.projection)
except Exception as e:
logger.exception(e)
if instance.srid:
instance.srid_url = "http://www.spatialreference.org/ref/" + \
instance.srid.replace(':', '/').lower() + "/"
elif instance.bbox_polygon is not None:
# Guessing 'EPSG:4326' by default
instance.srid = 'EPSG:4326'
else:
raise GeoNodeException(
"Invalid Projection. Layer is missing CRS!")
# Iterate over values from geoserver.
for key in ['alternate', 'store', 'storeType']:
# attr_name = key if 'typename' not in key else 'alternate'
# print attr_name
setattr(instance, key, values[key])
try:
if settings.RESOURCE_PUBLISHING:
if instance.is_published != gs_resource.advertised:
gs_resource.advertised = 'true'
if not settings.FREETEXT_KEYWORDS_READONLY:
# AF: Warning - this won't allow people to have empty keywords on GeoNode
if len(instance.keyword_list()
) == 0 and gs_resource.keywords:
for keyword in gs_resource.keywords:
if keyword not in instance.keyword_list():
instance.keywords.add(keyword)
if any(instance.keyword_list()):
keywords = instance.keyword_list()
gs_resource.keywords = [
kw for kw in list(set(keywords))
]
# gs_resource should only be called if
# ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED",
True):
gs_catalog.save(gs_resource)
except Exception as e:
msg = (
'Error while trying to save resource named %s in GeoServer, '
'try to use: "%s"' % (gs_resource, str(e)))
e.args = (msg, )
logger.exception(e)
# store the resource to avoid another geoserver call in the post_save
to_update = {
'title': instance.title or instance.name,
'abstract': instance.abstract or "",
'alternate': instance.alternate,
'bbox_polygon': instance.bbox_polygon,
'srid': 'EPSG:4326'
}
if is_monochromatic_image(instance.thumbnail_url):
to_update['thumbnail_url'] = staticfiles.static(
settings.MISSING_THUMBNAIL)
# Save all the modified information in the instance without triggering signals.
try:
with transaction.atomic():
ResourceBase.objects.filter(
id=instance.resourcebase_ptr.id).update(
**to_update)
# to_update['name'] = instance.name,
to_update[
'workspace'] = gs_resource.store.workspace.name
to_update['store'] = gs_resource.store.name
to_update['storeType'] = instance.storeType
to_update['typename'] = instance.alternate
Layer.objects.filter(id=instance.id).update(
**to_update)
# Refresh from DB
instance.refresh_from_db()
except Exception as e:
logger.exception(e)
# Refreshing CSW records
logger.debug(
f"... Updating the Catalogue entries for Layer {instance.title}"
)
try:
catalogue_post_save(instance=instance,
sender=instance.__class__)
except Exception as e:
logger.exception(e)
# Refreshing layer links
logger.debug(
f"... Creating Default Resource Links for Layer {instance.title}"
)
try:
set_resource_default_links(instance, instance, prune=True)
except Exception as e:
logger.exception(e)
# Save layer attributes
logger.debug(
f"... Refresh GeoServer attributes list for Layer {instance.title}"
)
try:
set_attributes_from_geoserver(instance)
except Exception as e:
logger.exception(e)
# Save layer styles
logger.debug(
f"... Refresh Legend links for Layer {instance.title}")
try:
set_styles(instance, gs_catalog)
except Exception as e:
logger.exception(e)
# Invalidate GeoWebCache for the updated resource
try:
_stylefilterparams_geowebcache_layer(instance.alternate)
_invalidate_geowebcache_layer(instance.alternate)
except Exception:
pass
# Creating Layer Thumbnail by sending a signal
from geonode.geoserver.signals import geoserver_post_save_complete
geoserver_post_save_complete.send(sender=instance.__class__,
instance=instance)
try:
geonode_upload_sessions = UploadSession.objects.filter(
resource=instance)
geonode_upload_sessions.update(processed=True)
except Exception as e:
logger.exception(e)
# Updating HAYSTACK Indexes if needed
if settings.HAYSTACK_SEARCH:
call_command('update_index')
def geoserver_finalize_upload(self, import_id, instance_id, permissions,
created, xml_file, sld_file, sld_uploaded,
tempdir):
"""
Finalize Layer and GeoServer configuration:
- Sets Layer Metadata from XML and updates GeoServer Layer accordingly.
- Sets Default Permissions
"""
instance = None
try:
instance = Layer.objects.get(id=instance_id)
except Layer.DoesNotExist:
logger.debug(f"Layer id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True:
from geonode.upload.models import Upload
upload = Upload.objects.get(import_id=import_id)
upload.layer = instance
upload.save()
try:
# Update the upload sessions
geonode_upload_sessions = UploadSession.objects.filter(
resource=instance)
geonode_upload_sessions.update(processed=False)
instance.upload_session = geonode_upload_sessions.first()
except Exception as e:
logger.exception(e)
# Sanity checks
if isinstance(xml_file, list):
if len(xml_file) > 0:
xml_file = xml_file[0]
else:
xml_file = None
elif not isinstance(xml_file, str):
xml_file = None
if xml_file and os.path.exists(xml_file) and os.access(
xml_file, os.R_OK):
instance.metadata_uploaded = True
try:
gs_resource = gs_catalog.get_resource(
name=instance.name,
store=instance.store,
workspace=instance.workspace)
except Exception:
try:
gs_resource = gs_catalog.get_resource(
name=instance.alternate,
store=instance.store,
workspace=instance.workspace)
except Exception:
try:
gs_resource = gs_catalog.get_resource(
name=instance.alternate or instance.typename)
except Exception:
gs_resource = None
if gs_resource:
# Updating GeoServer resource
gs_resource.title = instance.title
gs_resource.abstract = instance.abstract
gs_catalog.save(gs_resource)
if gs_resource.store:
instance.storeType = gs_resource.store.resource_type
if not instance.alternate:
instance.alternate = f"{gs_resource.store.workspace.name}:{gs_resource.name}"
if sld_uploaded:
geoserver_set_style(instance.id, sld_file)
else:
geoserver_create_style(instance.id, instance.name, sld_file,
tempdir)
logger.debug(
f'Finalizing (permissions and notifications) Layer {instance}')
instance.handle_moderated_uploads()
if permissions is not None:
logger.debug(
f'Setting permissions {permissions} for {instance.name}')
instance.set_permissions(permissions, created=created)
instance.save(notify=not created)
try:
logger.debug(
f"... Cleaning up the temporary folders {tempdir}")
if tempdir and os.path.exists(tempdir):
shutil.rmtree(tempdir)
except Exception as e:
logger.warning(e)
finally:
upload.complete = True
upload.save()
signals.upload_complete.send(sender=geoserver_finalize_upload,
layer=instance)
def geoserver_create_style(self, instance_id, name, sld_file, tempdir):
"""
Sets or create styles from Upload Session.
"""
instance = None
try:
instance = Layer.objects.get(id=instance_id)
except Layer.DoesNotExist:
logger.debug(f"Layer id {instance_id} does not exist yet!")
raise
lock_id = f'{self.request.id}'
with AcquireLock(lock_id) as lock:
if lock.acquire() is True and instance:
publishing = gs_catalog.get_layer(name)
if sld_file and os.path.exists(sld_file) and os.access(
sld_file, os.R_OK):
f = None
if os.path.isfile(sld_file):
try:
f = open(sld_file, 'r')
except Exception:
pass
elif tempdir and os.path.exists(tempdir):
if os.path.isfile(os.path.join(tempdir, sld_file)):
try:
f = open(os.path.join(tempdir, sld_file), 'r')
except Exception:
pass
if f:
sld = f.read()
f.close()
else:
sld = get_sld_for(gs_catalog, publishing)
else:
sld = get_sld_for(gs_catalog, publishing)
style = None
if sld is not None:
try:
gs_catalog.create_style(
name,
sld,
raw=True,
workspace=settings.DEFAULT_WORKSPACE)
gs_catalog.reset()
except geoserver.catalog.ConflictingDataError:
try:
gs_catalog.create_style(
name + '_layer',
sld,
raw=True,
workspace=settings.DEFAULT_WORKSPACE)
gs_catalog.reset()
except geoserver.catalog.ConflictingDataError as e:
msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % (
name, str(e))
logger.error(msg)
e.args = (msg, )
if style is None:
try:
style = gs_catalog.get_style(
name, workspace=settings.DEFAULT_WORKSPACE
) or gs_catalog.get_style(name)
except Exception:
logger.warn(
'Could not retreive the Layer default Style name')
try:
style = gs_catalog.get_style(name + '_layer', workspace=settings.DEFAULT_WORKSPACE) or \
gs_catalog.get_style(name + '_layer')
logger.warn(
'No style could be created for the layer, falling back to POINT default one'
)
except Exception as e:
style = gs_catalog.get_style('point')
logger.warn(str(e))
if style:
publishing.default_style = style
logger.debug('default style set to %s', name)
gs_catalog.save(publishing)
