def test_importer_configuration(self):
database_settings = self.DATABASE_DEFAULT_SETTINGS.copy()
ogc_server_settings = self.OGC_DEFAULT_SETTINGS.copy()
uploader_settings = self.UPLOADER_DEFAULT_SETTINGS.copy()
uploader_settings['BACKEND'] = 'geonode.importer'
self.assertTrue(['geonode_imports' not in database_settings.keys()])
# Test the importer backend without specifying a datastore or
# corresponding database.
with self.settings(UPLOADER=uploader_settings,
OGC_SERVER=ogc_server_settings,
DATABASES=database_settings):
OGC_Servers_Handler(ogc_server_settings)['default']
ogc_server_settings['default']['DATASTORE'] = 'geonode_imports'
# Test the importer backend with a datastore but no corresponding
# database.
with self.settings(UPLOADER=uploader_settings,
OGC_SERVER=ogc_server_settings,
DATABASES=database_settings):
OGC_Servers_Handler(ogc_server_settings)['default']
database_settings['geonode_imports'] = database_settings[
'default'].copy()
database_settings['geonode_imports'].update(
{'NAME': 'geonode_imports'})
# Test the importer backend with a datastore and a corresponding
# database, no exceptions should be thrown.
with self.settings(UPLOADER=uploader_settings,
OGC_SERVER=ogc_server_settings,
DATABASES=database_settings):
OGC_Servers_Handler(ogc_server_settings)['default']
def test_ogc_server_settings(self):
"""
Tests the OGC Servers Handler class.
"""
with override_settings(OGC_SERVER=self.OGC_DEFAULT_SETTINGS,
UPLOADER=self.UPLOADER_DEFAULT_SETTINGS):
OGC_SERVER = self.OGC_DEFAULT_SETTINGS.copy()
OGC_SERVER.update(
{'PUBLIC_LOCATION': 'http://geoserver:8080/geoserver/'})
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
default = OGC_SERVER.get('default')
self.assertEqual(ogc_settings.server, default)
self.assertEqual(ogc_settings.BACKEND, default.get('BACKEND'))
self.assertEqual(ogc_settings.LOCATION, default.get('LOCATION'))
self.assertEqual(ogc_settings.PUBLIC_LOCATION,
default.get('PUBLIC_LOCATION'))
self.assertEqual(ogc_settings.USER, default.get('USER'))
self.assertEqual(ogc_settings.PASSWORD, default.get('PASSWORD'))
self.assertEqual(ogc_settings.DATASTORE, '')
self.assertEqual(ogc_settings.credentials, ('admin', 'geoserver'))
self.assertTrue(ogc_settings.MAPFISH_PRINT_ENABLED)
self.assertTrue(ogc_settings.PRINT_NG_ENABLED)
self.assertTrue(ogc_settings.GEONODE_SECURITY_ENABLED)
self.assertFalse(ogc_settings.WMST_ENABLED)
self.assertTrue(ogc_settings.BACKEND_WRITE_ENABLED)
self.assertFalse(ogc_settings.WPS_ENABLED)
def test_ogc_server_defaults(self):
"""
Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings.
"""
OGC_SERVER = {
'default': dict(),
}
EXPECTATION = {
'default': {
'BACKEND': 'geonode.geoserver',
'LOCATION': 'http://localhost:8080/geoserver/',
'USER': '******',
'PASSWORD': '******',
'MAPFISH_PRINT_ENABLED': True,
'PRINTING_ENABLED': True,
'GEONODE_SECURITY_ENABLED': True,
'GEOGIT_ENABLED': False,
'WMST_ENABLED': False,
'BACKEND_WRITE_ENABLED': True,
'WPS_ENABLED': False,
'DATASTORE': str(),
'GEOGIT_DATASTORE_DIR': str(),
}
}
defaults = EXPECTATION.get('default')
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
self.assertEqual(ogc_settings.server, defaults)
self.assertEqual(ogc_settings.rest, defaults['LOCATION'] + 'rest')
self.assertEqual(ogc_settings.ows, defaults['LOCATION'] + 'ows')
# Make sure we get None vs a KeyError when the key does not exist
self.assertIsNone(ogc_settings.SFDSDFDSF)
def create_gs_thumbnail_geonode(instance, overwrite=False, check_bbox=False):
"""
Create a thumbnail with a GeoServer request.
"""
ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"]
wms_version = getattr(ogc_server_settings, "WMS_VERSION") or "1.1.1"
create_thumbnail(
instance,
wms_version=wms_version,
overwrite=overwrite,
)
def test_ogc_server_defaults(self):
"""
Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings.
"""
OGC_SERVER = {'default': dict()}
defaults = self.OGC_DEFAULT_SETTINGS.get('default')
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
self.assertEqual(ogc_settings.server, defaults)
self.assertEqual(ogc_settings.rest, defaults['LOCATION'] + 'rest')
self.assertEqual(ogc_settings.ows, defaults['LOCATION'] + 'ows')
# Make sure we get None vs a KeyError when the key does not exist
self.assertIsNone(ogc_settings.SFDSDFDSF)
def test_ogc_server_settings(self):
"""
Tests the OGC Servers Handler class.
"""
OGC_SERVER = {
'default': {
'BACKEND': 'geonode.geoserver',
'LOCATION': 'http://localhost:8080/geoserver/',
'PUBLIC_LOCATION': 'http://localhost:8080/geoserver/',
'USER': '******',
'PASSWORD': '******',
'MAPFISH_PRINT_ENABLED': True,
'PRINTING_ENABLED': True,
'GEONODE_SECURITY_ENABLED': True,
'GEOGIT_ENABLED': True,
'WMST_ENABLED': False,
'BACKEND_WRITE_ENABLED': True,
'WPS_ENABLED': False,
'DATASTORE': str(),
}
}
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
default = OGC_SERVER.get('default')
self.assertEqual(ogc_settings.server, default)
self.assertEqual(ogc_settings.BACKEND, default.get('BACKEND'))
self.assertEqual(ogc_settings.LOCATION, default.get('LOCATION'))
self.assertEqual(ogc_settings.PUBLIC_LOCATION,
default.get('PUBLIC_LOCATION'))
self.assertEqual(ogc_settings.USER, default.get('USER'))
self.assertEqual(ogc_settings.PASSWORD, default.get('PASSWORD'))
self.assertEqual(ogc_settings.DATASTORE, str())
self.assertEqual(ogc_settings.credentials, ('admin', 'geoserver'))
self.assertTrue(ogc_settings.MAPFISH_PRINT_ENABLED)
self.assertTrue(ogc_settings.PRINTING_ENABLED)
self.assertTrue(ogc_settings.GEONODE_SECURITY_ENABLED)
self.assertTrue(ogc_settings.GEOGIT_ENABLED)
self.assertFalse(ogc_settings.WMST_ENABLED)
self.assertTrue(ogc_settings.BACKEND_WRITE_ENABLED)
self.assertFalse(ogc_settings.WPS_ENABLED)
def test_ogc_server_defaults(self):
"""
Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings.
"""
from django.urls import reverse, resolve
from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities
OGC_SERVER = {'default': dict()}
defaults = self.OGC_DEFAULT_SETTINGS.get('default')
ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default']
self.assertEqual(ogc_settings.server, defaults)
self.assertEqual(ogc_settings.rest, f"{defaults['LOCATION']}rest")
self.assertEqual(ogc_settings.ows, f"{defaults['LOCATION']}ows")
# Make sure we get None vs a KeyError when the key does not exist
self.assertIsNone(ogc_settings.SFDSDFDSF)
# Testing REST endpoints
route = resolve('/gs/rest/layers').route
self.assertEqual(route, '^gs/rest/layers')
route = resolve('/gs/rest/imports').route
self.assertEqual(route, '^gs/rest/imports')
route = resolve('/gs/rest/sldservice').route
self.assertEqual(route, '^gs/rest/sldservice')
store_resolver = resolve('/gs/rest/stores/geonode_data/')
self.assertEqual(store_resolver.url_name, 'stores')
self.assertEqual(store_resolver.kwargs['store_type'], 'geonode_data')
self.assertEqual(store_resolver.route,
'^gs/rest/stores/(?P<store_type>\\w+)/$')
sld_resolver = resolve('/gs/rest/styles')
self.assertIsNone(sld_resolver.url_name)
self.assertTrue('workspace' not in sld_resolver.kwargs)
self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/styles')
self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/styles')
self.assertEqual(sld_resolver.route, '^gs/rest/styles')
sld_resolver = resolve('/gs/rest/workspaces/geonode/styles')
self.assertIsNone(sld_resolver.url_name)
self.assertEqual(sld_resolver.kwargs['workspace'], 'geonode')
self.assertEqual(sld_resolver.kwargs['proxy_path'],
'/gs/rest/workspaces')
self.assertEqual(sld_resolver.kwargs['downstream_path'],
'rest/workspaces')
self.assertEqual(sld_resolver.route,
'^gs/rest/workspaces/(?P<workspace>\\w+)')
# Testing OWS endpoints
wcs = _wcs_get_capabilities()
logger.debug(wcs)
self.assertIsNotNone(wcs)
try:
wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except Exception:
wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wcs.startswith(wcs_url))
self.assertIn("service=WCS", wcs)
self.assertIn("request=GetCapabilities", wcs)
self.assertIn("version=2.0.1", wcs)
wfs = _wfs_get_capabilities()
logger.debug(wfs)
self.assertIsNotNone(wfs)
try:
wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except Exception:
wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wfs.startswith(wfs_url))
self.assertIn("service=WFS", wfs)
self.assertIn("request=GetCapabilities", wfs)
self.assertIn("version=1.1.0", wfs)
wms = _wms_get_capabilities()
logger.debug(wms)
self.assertIsNotNone(wms)
try:
wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint'))
except Exception:
wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows')
self.assertTrue(wms.startswith(wms_url))
self.assertIn("service=WMS", wms)
self.assertIn("request=GetCapabilities", wms)
self.assertIn("version=1.3.0", wms)
# Test OWS Download Links
from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
instance = create_single_dataset("san_andres_y_providencia_water")
instance.name = 'san_andres_y_providencia_water'
instance.save()
bbox = instance.bbox
srid = instance.srid
height = 512
width = 512
# Default Style (expect exception since we are offline)
style = get_sld_for(gs_catalog, instance)
logger.error(
f" style -------------------------------------------> {style}")
if isinstance(style, str):
style = gs_catalog.get_style(instance.name,
workspace=instance.workspace)
self.assertIsNotNone(style)
self.assertFalse(isinstance(style, str))
instance.default_style, _ = Style.objects.get_or_create(
name=style.name,
defaults=dict(sld_title=style.sld_title, sld_body=style.sld_body))
self.assertIsNotNone(instance.default_style)
self.assertIsNotNone(instance.default_style.name)
# WMS Links
wms_links = wms_links(f"{ogc_settings.public_url}wms?",
instance.alternate, bbox, srid, height, width)
self.assertIsNotNone(wms_links)
self.assertEqual(len(wms_links), 3)
wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms')
identifier = urlencode({'layers': instance.alternate})
for _link in wms_links:
logger.debug(f'{wms_url} --> {_link[3]}')
self.assertTrue(wms_url in _link[3])
logger.debug(f'{identifier} --> {_link[3]}')
self.assertTrue(identifier in _link[3])
# WFS Links
wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?",
instance.alternate, bbox, srid)
self.assertIsNotNone(wfs_links)
self.assertEqual(len(wfs_links), 6)
wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs')
identifier = urlencode({'typename': instance.alternate})
for _link in wfs_links:
logger.debug(f'{wfs_url} --> {_link[3]}')
self.assertTrue(wfs_url in _link[3])
logger.debug(f'{identifier} --> {_link[3]}')
self.assertTrue(identifier in _link[3])
# WCS Links
wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?",
instance.alternate, bbox, srid)
self.assertIsNotNone(wcs_links)
self.assertEqual(len(wcs_links), 2)
wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs')
identifier = urlencode(
{'coverageid': instance.alternate.replace(':', '__', 1)})
for _link in wcs_links:
logger.debug(f'{wcs_url} --> {_link[3]}')
self.assertTrue(wcs_url in _link[3])
logger.debug(f'{identifier} --> {_link[3]}')
self.assertTrue(identifier in _link[3])
from ..maps.models import Map
from ..layers.models import Dataset
from ..documents.models import Document
from ..documents.enumerations import (
DOCUMENT_TYPE_MAP,
DOCUMENT_MIMETYPE_MAP)
from ..people.utils import get_valid_user
from ..layers.utils import resolve_regions
from ..layers.metadata import convert_keyword
from ..services.models import Service
from ..harvesting.models import HarvestableResource
logger = logging.getLogger(__name__)
ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)['default']
class KeywordHandler:
'''
Object needed to handle the keywords coming from the XML
The expected input are:
- instance (Dataset/Document/Map): instance of any object inherited from ResourceBase.
- keywords (list(dict)): Is required to analyze the keywords to find if some thesaurus is available.
'''
def __init__(self, instance, keywords):
self.instance = instance
self.keywords = keywords
def set_keywords(self):
def get_map(
ogc_server_location: str,
layers: List,
bbox: List,
wms_version: str = settings.OGC_SERVER["default"].get("WMS_VERSION", "1.1.1"),
mime_type: str = "image/png",
styles: List = None,
width: int = 240,
height: int = 200,
max_retries: int = 3,
retry_delay: int = 1,
):
"""
Function fetching an image from OGC server.
For the requests to the configured OGC backend (ogc_server_settings.LOCATION) the function tries to generate
an access_token and attach it to the URL.
If access_token is not added ant the request is against Geoserver Basic Authentication is used instead.
If image retrieval fails, function retries to fetch the image max_retries times, waiting
retry_delay seconds between consecutive requests.
:param ogc_server_location: OGC server URL
:param layers: layers which should be fetched from the OGC server
:param bbox: area's bounding box in format: [west, east, south, north, CRS]
:param wms_version: WMS version of the query (default: 1.1.1)
:param mime_type: mime type of the returned image
:param styles: styles, which OGC server should use for rendering an image
:param width: width of the returned image
:param height: height of the returned image
:param max_retries: maximum number of retries before skipping retrieval
:param retry_delay: number of seconds waited between retries
:returns: retrieved image
"""
ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"]
if ogc_server_location is not None:
thumbnail_url = ogc_server_location
else:
thumbnail_url = ogc_server_settings.LOCATION
if thumbnail_url.startswith(ogc_server_settings.PUBLIC_LOCATION):
thumbnail_url = thumbnail_url.replace(ogc_server_settings.PUBLIC_LOCATION, ogc_server_settings.LOCATION)
wms_endpoint = ""
additional_kwargs = {}
if thumbnail_url == ogc_server_settings.LOCATION:
# add access token to requests to Geoserver (logic based on the previous implementation)
username = ogc_server_settings.credentials.username
user = get_user_model().objects.filter(username=username).first()
if user:
access_token = get_or_create_token(user)
if access_token and not access_token.is_expired():
additional_kwargs['access_token'] = access_token.token
# add WMS endpoint to requests to Geoserver
wms_endpoint = getattr(ogc_server_settings, "WMS_ENDPOINT") or "ows"
# prepare authorization for WMS service
headers = {}
if thumbnail_url.startswith(ogc_server_settings.LOCATION):
if "access_token" not in additional_kwargs.keys():
# for the Geoserver backend, use Basic Auth, if access_token is not provided
_user, _pwd = ogc_server_settings.credentials
encoded_credentials = base64.b64encode(f"{_user}:{_pwd}".encode("UTF-8")).decode("ascii")
headers["Authorization"] = f"Basic {encoded_credentials}"
else:
headers["Authorization"] = f"Bearer {additional_kwargs['access_token']}"
wms = WebMapService(
f"{thumbnail_url}{wms_endpoint}",
version=wms_version,
headers=headers)
image = None
for retry in range(max_retries):
try:
# fetch data
image = wms.getmap(
layers=layers,
styles=styles,
srs=bbox[-1] if bbox else None,
bbox=[bbox[0], bbox[2], bbox[1], bbox[3]] if bbox else None,
size=(width, height),
format=mime_type,
transparent=True,
timeout=getattr(ogc_server_settings, "TIMEOUT", None),
**additional_kwargs,
)
# validate response
if not image or "ServiceException" in str(image.read()):
raise ThumbnailError(
f"Fetching partial thumbnail from {thumbnail_url} failed with response: {str(image)}"
)
except Exception as e:
if retry + 1 >= max_retries:
logger.exception(e)
return
time.sleep(retry_delay)
continue
else:
break
return image.read()
def _datasets_locations(
instance: Union[Dataset, Map],
compute_bbox: bool = False,
target_crs: str = "EPSG:3857") -> Tuple[List[List], List]:
"""
Function returning a list mapping instance's datasets to their locations, enabling to construct a minimum
number of WMS request for multiple datasets of the same OGC source (ensuring datasets order for Maps)
:param instance: instance of Dataset or Map models
:param compute_bbox: flag determining whether a BBOX containing the instance should be computed,
based on instance's datasets
:param target_crs: valid only when compute_bbox is True - CRS of the returned BBOX
:return: a tuple with a list, which maps datasets to their locations in a correct datasets order
e.g.
[
["http://localhost:8080/geoserver/": ["geonode:layer1", "geonode:layer2]]
]
and a list optionally consisting of 5 elements containing west, east, south, north
instance's boundaries and CRS
"""
ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"]
locations = []
bbox = []
if isinstance(instance, Dataset):
locations.append([
instance.ows_url or ogc_server_settings.LOCATION,
[instance.alternate], []
])
if compute_bbox:
if instance.ll_bbox_polygon:
bbox = utils.clean_bbox(instance.ll_bbox, target_crs)
elif (instance.bbox[-1].upper() != 'EPSG:3857'
and target_crs.upper() == 'EPSG:3857'
and utils.exceeds_epsg3857_area_of_use(instance.bbox)):
# handle exceeding the area of use of the default thumb's CRS
bbox = utils.transform_bbox(
utils.crop_to_3857_area_of_use(instance.bbox), target_crs)
else:
bbox = utils.transform_bbox(instance.bbox, target_crs)
elif isinstance(instance, Map):
for map_dataset in instance.maplayers.iterator():
if not map_dataset.local and not map_dataset.ows_url:
logger.warning(
"Incorrectly defined remote dataset encountered (no OWS URL defined)."
"Skipping it in the thumbnail generation.")
continue
name = get_dataset_name(map_dataset)
store = map_dataset.store
workspace = get_dataset_workspace(map_dataset)
map_dataset_style = map_dataset.current_style
if store and Dataset.objects.filter(
store=store, workspace=workspace, name=name).count() > 0:
dataset = Dataset.objects.filter(store=store,
workspace=workspace,
name=name).first()
elif workspace and Dataset.objects.filter(workspace=workspace,
name=name).count() > 0:
dataset = Dataset.objects.filter(workspace=workspace,
name=name).first()
elif Dataset.objects.filter(
alternate=map_dataset.name).count() > 0:
dataset = Dataset.objects.filter(
alternate=map_dataset.name).first()
else:
logger.warning(
f"Dataset for MapLayer {name} was not found. Skipping it in the thumbnail."
)
continue
if dataset.subtype in ['tileStore', 'remote']:
# limit number of locations, ensuring dataset order
if len(locations) and locations[-1][
0] == dataset.remote_service.service_url:
# if previous dataset's location is the same as the current one - append current dataset there
locations[-1][1].append(dataset.alternate)
# update the styles too
if map_dataset_style:
locations[-1][2].append(map_dataset_style)
else:
locations.append([
dataset.remote_service.service_url,
[dataset.alternate],
[map_dataset_style] if map_dataset_style else []
])
else:
# limit number of locations, ensuring dataset order
if len(locations) and locations[-1][0] == settings.OGC_SERVER[
"default"]["LOCATION"]:
# if previous dataset's location is the same as the current one - append current dataset there
locations[-1][1].append(dataset.alternate)
# update the styles too
if map_dataset_style:
locations[-1][2].append(map_dataset_style)
else:
locations.append([
settings.OGC_SERVER["default"]["LOCATION"],
[dataset.alternate],
[map_dataset_style] if map_dataset_style else []
])
if compute_bbox:
if dataset.ll_bbox_polygon:
dataset_bbox = utils.clean_bbox(dataset.ll_bbox,
target_crs)
elif (dataset.bbox[-1].upper() != 'EPSG:3857'
and target_crs.upper() == 'EPSG:3857'
and utils.exceeds_epsg3857_area_of_use(dataset.bbox)):
# handle exceeding the area of use of the default thumb's CRS
dataset_bbox = utils.transform_bbox(
utils.crop_to_3857_area_of_use(dataset.bbox),
target_crs)
else:
dataset_bbox = utils.transform_bbox(
dataset.bbox, target_crs)
if not bbox:
bbox = dataset_bbox
else:
# dataset's BBOX: (left, right, bottom, top)
bbox = [
min(bbox[0], dataset_bbox[0]),
max(bbox[1], dataset_bbox[1]),
min(bbox[2], dataset_bbox[2]),
max(bbox[3], dataset_bbox[3]),
]
if bbox and len(bbox) < 5:
bbox = list(bbox) + [target_crs] # convert bbox to list, if it's tuple
return locations, bbox