def saveToGeonode(payload):
admin = Profile.objects.filter(is_superuser=True).first()
logger.debug('Uploading layer %s to geonode' % payload['zip_name'])
# use overwrite to make sure if a layer exists it gets updated
uploaded = upload(payload['zipfile_path'], admin, overwrite=True)[0]
if uploaded['status'] == 'failed':
raise Exception("Failed layer %s with error %s " %
(payload['zip_name'], uploaded['error']))
else:
uploaded_name = uploaded['name']
if any(s in payload['zip_name'] for s in ['_a', '_p', '_l']):
gs_layer = gs_catalog.get_layer(name=uploaded_name)
gs_style = gs_catalog.get_style(name=payload['style_name'])
if not gs_style:
#let's make sure the style is there
gs_catalog.create_style(payload['style_name'],
payload['style_body'])
gs_style = gs_catalog.get_style(name=payload['style_name'])
gs_layer.default_style = gs_style
gs_catalog.save(gs_layer)
# gs_catalog.reload()
saved_layer = Layer.objects.get(name=uploaded_name)
mapSetLayer = createMapSetLayer(saved_layer, payload['zip_name'])
payload['mapset'].layers.add(mapSetLayer)
#create_gs_thumbnail(saved_layer, overwrite=True)
return uploaded_name
def get_new_style_name(self, sld_name):
sld_name = SLUGIFIER(sld_name)
style = gs_catalog.get_style(
sld_name, workspace=settings.DEFAULT_WORKSPACE)
if not style:
return sld_name
else:
timestr = time.strftime("%Y%m%d_%H%M%S")
return "{}_{}".format(sld_name, timestr)
def set_default_style_to_open_in_visual_mode(instance, **kwargs):
if isinstance(instance, Dataset):
style = gs_catalog.get_style(instance.name, workspace=instance.workspace) or \
gs_catalog.get_style(isinstance.name)
if style:
headers = {
"Content-type": "application/json",
"Accept": "application/json"
}
data = {"style": {"metadata": {"msForceVisual": "true"}}}
body_href = os.path.splitext(style.body_href)[0] + '.json'
resp = gs_catalog.http_request(body_href,
method='put',
data=json.dumps(data),
headers=headers)
if resp.status_code not in (200, 201, 202):
raise FailedRequestError(
'Failed to update style {} : {}, {}'.format(
style.name, resp.status_code, resp.text))
def create_style(self, name, sld_body, overwrite=True, raw=True):
name = self.get_new_style_name(name)
sld_body = self.convert_sld_attributes(sld_body)
gs_catalog.create_style(name,
sld_body,
overwrite=overwrite,
raw=True,
workspace=settings.DEFAULT_WORKSPACE)
style = gs_catalog.get_style(name,
workspace=settings.DEFAULT_WORKSPACE)
return style
def test_ogc_server_defaults(self):
"""
Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings.
"""
from django.urls import reverse, resolve
from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities
OGC_SERVER = {'default': dict()}
defaults = self.OGC_DEFAULT_SETTINGS.get('default')
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
self.assertEqual(ogc_settings.server, defaults)
self.assertEqual(ogc_settings.rest, f"{defaults['LOCATION']}rest")
self.assertEqual(ogc_settings.ows, f"{defaults['LOCATION']}ows")
# Make sure we get None vs a KeyError when the key does not exist
self.assertIsNone(ogc_settings.SFDSDFDSF)
# Testing REST endpoints
route = resolve('/gs/rest/layers').route
self.assertEqual(route, '^gs/rest/layers')
route = resolve('/gs/rest/imports').route
self.assertEqual(route, '^gs/rest/imports')
route = resolve('/gs/rest/sldservice').route
self.assertEqual(route, '^gs/rest/sldservice')
store_resolver = resolve('/gs/rest/stores/geonode_data/')
self.assertEqual(store_resolver.url_name, 'stores')
self.assertEqual(store_resolver.kwargs['store_type'], 'geonode_data')
self.assertEqual(store_resolver.route,
'^gs/rest/stores/(?P<store_type>\\w+)/$')
sld_resolver = resolve('/gs/rest/styles')
self.assertIsNone(sld_resolver.url_name)
self.assertTrue('workspace' not in sld_resolver.kwargs)
self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/styles')
self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/styles')
self.assertEqual(sld_resolver.route, '^gs/rest/styles')
sld_resolver = resolve('/gs/rest/workspaces/geonode/styles')
self.assertIsNone(sld_resolver.url_name)
self.assertEqual(sld_resolver.kwargs['workspace'], 'geonode')
self.assertEqual(sld_resolver.kwargs['proxy_path'],
'/gs/rest/workspaces')
self.assertEqual(sld_resolver.kwargs['downstream_path'],
'rest/workspaces')
self.assertEqual(sld_resolver.route,
'^gs/rest/workspaces/(?P<workspace>\\w+)')
# Testing OWS endpoints
wcs = _wcs_get_capabilities()
logger.debug(wcs)
self.assertIsNotNone(wcs)
try:
wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except Exception:
wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wcs.startswith(wcs_url))
self.assertIn("service=WCS", wcs)
self.assertIn("request=GetCapabilities", wcs)
self.assertIn("version=2.0.1", wcs)
wfs = _wfs_get_capabilities()
logger.debug(wfs)
self.assertIsNotNone(wfs)
try:
wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except Exception:
wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wfs.startswith(wfs_url))
self.assertIn("service=WFS", wfs)
self.assertIn("request=GetCapabilities", wfs)
self.assertIn("version=1.1.0", wfs)
wms = _wms_get_capabilities()
logger.debug(wms)
self.assertIsNotNone(wms)
try:
wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except Exception:
wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wms.startswith(wms_url))
self.assertIn("service=WMS", wms)
self.assertIn("request=GetCapabilities", wms)
self.assertIn("version=1.3.0", wms)
# Test OWS Download Links
from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
instance = create_single_dataset("san_andres_y_providencia_water")
instance.name = 'san_andres_y_providencia_water'
instance.save()
bbox = instance.bbox
srid = instance.srid
height = 512
width = 512
# Default Style (expect exception since we are offline)
style = get_sld_for(gs_catalog, instance)
logger.error(
f" style -------------------------------------------> {style}")
if isinstance(style, str):
style = gs_catalog.get_style(instance.name,
workspace=instance.workspace)
self.assertIsNotNone(style)
self.assertFalse(isinstance(style, str))
instance.default_style, _ = Style.objects.get_or_create(
name=style.name,
defaults=dict(sld_title=style.sld_title, sld_body=style.sld_body))
self.assertIsNotNone(instance.default_style)
self.assertIsNotNone(instance.default_style.name)
# WMS Links
wms_links = wms_links(f"{ogc_settings.public_url}wms?",
instance.alternate, bbox, srid, height, width)
self.assertIsNotNone(wms_links)
self.assertEqual(len(wms_links), 3)
wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms')
identifier = urlencode({'layers': instance.alternate})
for _link in wms_links:
logger.debug(f'{wms_url} --> {_link[3]}')
self.assertTrue(wms_url in _link[3])
logger.debug(f'{identifier} --> {_link[3]}')
self.assertTrue(identifier in _link[3])
# WFS Links
wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?",
instance.alternate, bbox, srid)
self.assertIsNotNone(wfs_links)
self.assertEqual(len(wfs_links), 6)
wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs')
identifier = urlencode({'typename': instance.alternate})
for _link in wfs_links:
logger.debug(f'{wfs_url} --> {_link[3]}')
self.assertTrue(wfs_url in _link[3])
logger.debug(f'{identifier} --> {_link[3]}')
self.assertTrue(identifier in _link[3])
# WCS Links
wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?",
instance.alternate, bbox, srid)
self.assertIsNotNone(wcs_links)
self.assertEqual(len(wcs_links), 2)
wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs')
identifier = urlencode(
{'coverageid': instance.alternate.replace(':', '__', 1)})
for _link in wcs_links:
logger.debug(f'{wcs_url} --> {_link[3]}')
self.assertTrue(wcs_url in _link[3])
logger.debug(f'{identifier} --> {_link[3]}')
self.assertTrue(identifier in _link[3])
def test_ogc_server_defaults(self):
"""
Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings.
"""
OGC_SERVER = {'default': dict()}
defaults = self.OGC_DEFAULT_SETTINGS.get('default')
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
self.assertEqual(ogc_settings.server, defaults)
self.assertEqual(ogc_settings.rest, defaults['LOCATION'] + 'rest')
self.assertEqual(ogc_settings.ows, defaults['LOCATION'] + 'ows')
# Make sure we get None vs a KeyError when the key does not exist
self.assertIsNone(ogc_settings.SFDSDFDSF)
# Testing OWS endpoints
from django.urls import reverse
from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities
wcs = _wcs_get_capabilities()
logger.debug(wcs)
self.assertIsNotNone(wcs)
try:
wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except BaseException:
wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wcs.startswith(wcs_url))
self.assertIn("service=WCS", wcs)
self.assertIn("request=GetCapabilities", wcs)
self.assertIn("version=2.0.1", wcs)
wfs = _wfs_get_capabilities()
logger.debug(wfs)
self.assertIsNotNone(wfs)
try:
wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except BaseException:
wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wfs.startswith(wfs_url))
self.assertIn("service=WFS", wfs)
self.assertIn("request=GetCapabilities", wfs)
self.assertIn("version=1.1.0", wfs)
wms = _wms_get_capabilities()
logger.debug(wms)
self.assertIsNotNone(wms)
try:
wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except BaseException:
wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wms.startswith(wms_url))
self.assertIn("service=WMS", wms)
self.assertIn("request=GetCapabilities", wms)
self.assertIn("version=1.3.0", wms)
# Test OWS Download Links
from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
instance = Layer.objects.all()[0]
bbox = instance.bbox
srid = instance.srid
height = 512
width = 512
# Default Style (expect exception since we are offline)
style = None
with self.assertRaises(GeoNodeException):
style = get_sld_for(gs_catalog, instance)
self.assertIsNone(style)
style = gs_catalog.get_style("line")
self.assertIsNotNone(style)
instance.default_style, _ = Style.objects.get_or_create(
name=style.name,
defaults=dict(
sld_title=style.sld_title,
sld_body=style.sld_body
)
)
self.assertIsNotNone(instance.default_style)
self.assertIsNotNone(instance.default_style.name)
# WMS Links
wms_links = wms_links(ogc_settings.public_url + 'wms?',
instance.alternate,
bbox,
srid,
height,
width)
self.assertIsNotNone(wms_links)
self.assertEquals(len(wms_links), 3)
wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms')
identifier = urlencode({'layers': instance.alternate})
for _link in wms_links:
logger.debug('%s --> %s' % (wms_url, _link[3]))
self.assertTrue(wms_url in _link[3])
logger.debug('%s --> %s' % (identifier, _link[3]))
self.assertTrue(identifier in _link[3])
# WFS Links
wfs_links = wfs_links(ogc_settings.public_url + 'wfs?',
instance.alternate,
bbox,
srid)
self.assertIsNotNone(wfs_links)
self.assertEquals(len(wfs_links), 6)
wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs')
identifier = urlencode({'typename': instance.alternate})
for _link in wfs_links:
logger.debug('%s --> %s' % (wfs_url, _link[3]))
self.assertTrue(wfs_url in _link[3])
logger.debug('%s --> %s' % (identifier, _link[3]))
self.assertTrue(identifier in _link[3])
# WCS Links
wcs_links = wcs_links(ogc_settings.public_url + 'wcs?',
instance.alternate,
bbox,
srid)
self.assertIsNotNone(wcs_links)
self.assertEquals(len(wcs_links), 2)
wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs')
identifier = urlencode({'coverageid': instance.alternate})
for _link in wcs_links:
logger.debug('%s --> %s' % (wcs_url, _link[3]))
self.assertTrue(wcs_url in _link[3])
logger.debug('%s --> %s' % (identifier, _link[3]))
self.assertTrue(identifier in _link[3])
# Thumbnails Generation Default
create_gs_thumbnail(instance, overwrite=True)
self.assertIsNotNone(instance.get_thumbnail_url())
# Thumbnails Generation Through "remote url"
create_gs_thumbnail_geonode(instance, overwrite=True, check_bbox=True)
# Thumbnails Generation Through "image"
request_body = {
'width': width,
'height': height,
'layers': instance.alternate
}
if hasattr(instance, 'default_style'):
if instance.default_style:
request_body['styles'] = instance.default_style.name
self.assertIsNotNone(request_body['styles'])
try:
image = _prepare_thumbnail_body_from_opts(request_body)
except BaseException as e:
logger.exception(e)
image = None
# We are offline here, the layer does not exists in GeoServer
# - we expect the image is None
self.assertIsNone(image)
def set_default_layer_style(self, layername, stylename):
gs_layer = gs_catalog.get_layer(layername)
gs_layer.default_style = gs_catalog.get_style(
stylename, workspace=settings.DEFAULT_WORKSPACE)
gs_catalog.save(gs_layer)
def file_upload(filename,
name=None,
user=None,
title=None,
abstract=None,
style_name=None,
skip=True,
overwrite=False,
keywords=[],
charset='UTF-8',
category=None,
date=None):
"""Saves a layer in GeoNode asking as little information as possible.
Only filename is required, user and title are optional.
"""
# Get a valid user
theuser = get_valid_user(user)
# Create a new upload session
upload_session = UploadSession.objects.create(user=theuser)
# Get all the files uploaded with the layer
files = get_files(filename)
# Set a default title that looks nice ...
if title is None:
basename = os.path.splitext(os.path.basename(filename))[0]
title = basename.title().replace('_', ' ')
# Create a name from the title if it is not passed.
if name is None:
name = slugify(title).replace('-', '_')
if category is not None:
category = TopicCategory.objects.get(identifier=category)
# Generate a name that is not taken if overwrite is False.
valid_name = get_valid_layer_name(name, overwrite)
# Add them to the upload session (new file fields are created).
assigned_name = None
for type_name, fn in files.items():
with open(fn, 'rb') as f:
upload_session.layerfile_set.create(
name=type_name,
file=File(f,
name='%s.%s' %
(assigned_name or valid_name, type_name)))
# save the system assigned name for the remaining files
if not assigned_name:
the_file = upload_session.layerfile_set.all()[0].file.name
assigned_name = os.path.splitext(os.path.basename(the_file))[0]
# Get a bounding box
bbox_x0, bbox_x1, bbox_y0, bbox_y1 = get_bbox(filename)
# by default, if RESOURCE_PUBLISHING=True then layer.is_published
# must be set to False
is_published = True
if settings.RESOURCE_PUBLISHING:
is_published = False
defaults = {
'upload_session': upload_session,
'title': title,
'abstract': abstract,
'owner': user,
'charset': charset,
'bbox_x0': bbox_x0,
'bbox_x1': bbox_x1,
'bbox_y0': bbox_y0,
'bbox_y1': bbox_y1,
'is_published': is_published,
'category': category
}
# set metadata
if 'xml' in files:
xml_file = open(files['xml'])
defaults['metadata_uploaded'] = True
# get model properties from XML
vals, keywords = set_metadata(xml_file.read())
for key, value in vals.items():
if key == 'spatial_representation_type':
value = SpatialRepresentationType(identifier=value)
elif key == 'topic_category':
value, created = TopicCategory.objects.get_or_create(
identifier=value.lower(),
defaults={
'description': '',
'gn_description': value
})
key = 'category'
defaults[key] = value
else:
defaults[key] = value
# If it is a vector file, create the layer in postgis.
if is_vector(filename):
defaults['storeType'] = 'dataStore'
# If it is a raster file, get the resolution.
if is_raster(filename):
defaults['storeType'] = 'coverageStore'
# Create a Django object.
layer, created = Layer.objects.get_or_create(name=valid_name,
defaults=defaults)
# set layer style in geoserver
if style_name is not None:
# import here because import at top of file fails for some reason
from geonode.geoserver.helpers import gs_catalog
try:
gs_style = gs_catalog.get_style(style_name)
except:
gs_style = None
if gs_style is not None:
gs_layer = gs_catalog.get_layer(layer.name)
gs_layer.default_style = gs_style
gs_catalog.save(gs_layer)
# set layer date
if date is not None:
layer.date = date
# Delete the old layers if overwrite is true
# and the layer was not just created
# process the layer again after that by
# doing a layer.save()
if not created and overwrite:
layer.upload_session.layerfile_set.all().delete()
layer.upload_session = upload_session
# Pass the parameter overwrite to tell whether the
# geoserver_post_save_signal should upload the new file or not
layer.overwrite = overwrite
layer.save()
# Assign the keywords (needs to be done after saving)
if len(keywords) > 0:
layer.keywords.add(*keywords)
return layer
