def test_importer_configuration(self): database_settings = self.DATABASE_DEFAULT_SETTINGS.copy() ogc_server_settings = self.OGC_DEFAULT_SETTINGS.copy() uploader_settings = self.UPLOADER_DEFAULT_SETTINGS.copy() uploader_settings['BACKEND'] = 'geonode.importer' self.assertTrue(['geonode_imports' not in database_settings.keys()]) # Test the importer backend without specifying a datastore or # corresponding database. with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): OGC_Servers_Handler(ogc_server_settings)['default'] ogc_server_settings['default']['DATASTORE'] = 'geonode_imports' # Test the importer backend with a datastore but no corresponding # database. with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): OGC_Servers_Handler(ogc_server_settings)['default'] database_settings['geonode_imports'] = database_settings[ 'default'].copy() database_settings['geonode_imports'].update( {'NAME': 'geonode_imports'}) # Test the importer backend with a datastore and a corresponding # database, no exceptions should be thrown. with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): OGC_Servers_Handler(ogc_server_settings)['default']
def test_ogc_server_settings(self): """ Tests the OGC Servers Handler class. """ with override_settings(OGC_SERVER=self.OGC_DEFAULT_SETTINGS, UPLOADER=self.UPLOADER_DEFAULT_SETTINGS): OGC_SERVER = self.OGC_DEFAULT_SETTINGS.copy() OGC_SERVER.update( {'PUBLIC_LOCATION': 'http://geoserver:8080/geoserver/'}) ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] default = OGC_SERVER.get('default') self.assertEqual(ogc_settings.server, default) self.assertEqual(ogc_settings.BACKEND, default.get('BACKEND')) self.assertEqual(ogc_settings.LOCATION, default.get('LOCATION')) self.assertEqual(ogc_settings.PUBLIC_LOCATION, default.get('PUBLIC_LOCATION')) self.assertEqual(ogc_settings.USER, default.get('USER')) self.assertEqual(ogc_settings.PASSWORD, default.get('PASSWORD')) self.assertEqual(ogc_settings.DATASTORE, '') self.assertEqual(ogc_settings.credentials, ('admin', 'geoserver')) self.assertTrue(ogc_settings.MAPFISH_PRINT_ENABLED) self.assertTrue(ogc_settings.PRINT_NG_ENABLED) self.assertTrue(ogc_settings.GEONODE_SECURITY_ENABLED) self.assertFalse(ogc_settings.WMST_ENABLED) self.assertTrue(ogc_settings.BACKEND_WRITE_ENABLED) self.assertFalse(ogc_settings.WPS_ENABLED)
def test_ogc_server_defaults(self): """ Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings. """ OGC_SERVER = { 'default': dict(), } EXPECTATION = { 'default': { 'BACKEND': 'geonode.geoserver', 'LOCATION': 'http://localhost:8080/geoserver/', 'USER': '******', 'PASSWORD': '******', 'MAPFISH_PRINT_ENABLED': True, 'PRINTING_ENABLED': True, 'GEONODE_SECURITY_ENABLED': True, 'GEOGIT_ENABLED': False, 'WMST_ENABLED': False, 'BACKEND_WRITE_ENABLED': True, 'WPS_ENABLED': False, 'DATASTORE': str(), 'GEOGIT_DATASTORE_DIR': str(), } } defaults = EXPECTATION.get('default') ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] self.assertEqual(ogc_settings.server, defaults) self.assertEqual(ogc_settings.rest, defaults['LOCATION'] + 'rest') self.assertEqual(ogc_settings.ows, defaults['LOCATION'] + 'ows') # Make sure we get None vs a KeyError when the key does not exist self.assertIsNone(ogc_settings.SFDSDFDSF)
def create_gs_thumbnail_geonode(instance, overwrite=False, check_bbox=False): """ Create a thumbnail with a GeoServer request. """ ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] wms_version = getattr(ogc_server_settings, "WMS_VERSION") or "1.1.1" create_thumbnail( instance, wms_version=wms_version, overwrite=overwrite, )
def test_ogc_server_defaults(self): """ Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings. """ OGC_SERVER = {'default': dict()} defaults = self.OGC_DEFAULT_SETTINGS.get('default') ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] self.assertEqual(ogc_settings.server, defaults) self.assertEqual(ogc_settings.rest, defaults['LOCATION'] + 'rest') self.assertEqual(ogc_settings.ows, defaults['LOCATION'] + 'ows') # Make sure we get None vs a KeyError when the key does not exist self.assertIsNone(ogc_settings.SFDSDFDSF)
def test_ogc_server_settings(self): """ Tests the OGC Servers Handler class. """ OGC_SERVER = { 'default': { 'BACKEND': 'geonode.geoserver', 'LOCATION': 'http://localhost:8080/geoserver/', 'PUBLIC_LOCATION': 'http://localhost:8080/geoserver/', 'USER': '******', 'PASSWORD': '******', 'MAPFISH_PRINT_ENABLED': True, 'PRINTING_ENABLED': True, 'GEONODE_SECURITY_ENABLED': True, 'GEOGIT_ENABLED': True, 'WMST_ENABLED': False, 'BACKEND_WRITE_ENABLED': True, 'WPS_ENABLED': False, 'DATASTORE': str(), } } ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] default = OGC_SERVER.get('default') self.assertEqual(ogc_settings.server, default) self.assertEqual(ogc_settings.BACKEND, default.get('BACKEND')) self.assertEqual(ogc_settings.LOCATION, default.get('LOCATION')) self.assertEqual(ogc_settings.PUBLIC_LOCATION, default.get('PUBLIC_LOCATION')) self.assertEqual(ogc_settings.USER, default.get('USER')) self.assertEqual(ogc_settings.PASSWORD, default.get('PASSWORD')) self.assertEqual(ogc_settings.DATASTORE, str()) self.assertEqual(ogc_settings.credentials, ('admin', 'geoserver')) self.assertTrue(ogc_settings.MAPFISH_PRINT_ENABLED) self.assertTrue(ogc_settings.PRINTING_ENABLED) self.assertTrue(ogc_settings.GEONODE_SECURITY_ENABLED) self.assertTrue(ogc_settings.GEOGIT_ENABLED) self.assertFalse(ogc_settings.WMST_ENABLED) self.assertTrue(ogc_settings.BACKEND_WRITE_ENABLED) self.assertFalse(ogc_settings.WPS_ENABLED)
def test_ogc_server_defaults(self): """ Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings. """ from django.urls import reverse, resolve from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities OGC_SERVER = {'default': dict()} defaults = self.OGC_DEFAULT_SETTINGS.get('default') ogc_settings = OGC_Servers_Handler(OGC_SERVER)['default'] self.assertEqual(ogc_settings.server, defaults) self.assertEqual(ogc_settings.rest, f"{defaults['LOCATION']}rest") self.assertEqual(ogc_settings.ows, f"{defaults['LOCATION']}ows") # Make sure we get None vs a KeyError when the key does not exist self.assertIsNone(ogc_settings.SFDSDFDSF) # Testing REST endpoints route = resolve('/gs/rest/layers').route self.assertEqual(route, '^gs/rest/layers') route = resolve('/gs/rest/imports').route self.assertEqual(route, '^gs/rest/imports') route = resolve('/gs/rest/sldservice').route self.assertEqual(route, '^gs/rest/sldservice') store_resolver = resolve('/gs/rest/stores/geonode_data/') self.assertEqual(store_resolver.url_name, 'stores') self.assertEqual(store_resolver.kwargs['store_type'], 'geonode_data') self.assertEqual(store_resolver.route, '^gs/rest/stores/(?P<store_type>\\w+)/$') sld_resolver = resolve('/gs/rest/styles') self.assertIsNone(sld_resolver.url_name) self.assertTrue('workspace' not in sld_resolver.kwargs) self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/styles') self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/styles') self.assertEqual(sld_resolver.route, '^gs/rest/styles') sld_resolver = resolve('/gs/rest/workspaces/geonode/styles') self.assertIsNone(sld_resolver.url_name) self.assertEqual(sld_resolver.kwargs['workspace'], 'geonode') self.assertEqual(sld_resolver.kwargs['proxy_path'], '/gs/rest/workspaces') self.assertEqual(sld_resolver.kwargs['downstream_path'], 'rest/workspaces') self.assertEqual(sld_resolver.route, '^gs/rest/workspaces/(?P<workspace>\\w+)') # Testing OWS endpoints wcs = _wcs_get_capabilities() logger.debug(wcs) self.assertIsNotNone(wcs) try: wcs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except Exception: wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wcs.startswith(wcs_url)) self.assertIn("service=WCS", wcs) self.assertIn("request=GetCapabilities", wcs) self.assertIn("version=2.0.1", wcs) wfs = _wfs_get_capabilities() logger.debug(wfs) self.assertIsNotNone(wfs) try: wfs_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except Exception: wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wfs.startswith(wfs_url)) self.assertIn("service=WFS", wfs) self.assertIn("request=GetCapabilities", wfs) self.assertIn("version=1.1.0", wfs) wms = _wms_get_capabilities() logger.debug(wms) self.assertIsNotNone(wms) try: wms_url = urljoin(settings.SITEURL, reverse('ows_endpoint')) except Exception: wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'ows') self.assertTrue(wms.startswith(wms_url)) self.assertIn("service=WMS", wms) self.assertIn("request=GetCapabilities", wms) self.assertIn("version=1.3.0", wms) # Test OWS Download Links from geonode.geoserver.ows import wcs_links, wfs_links, wms_links instance = create_single_dataset("san_andres_y_providencia_water") instance.name = 'san_andres_y_providencia_water' instance.save() bbox = instance.bbox srid = instance.srid height = 512 width = 512 # Default Style (expect exception since we are offline) style = get_sld_for(gs_catalog, instance) logger.error( f" style -------------------------------------------> {style}") if isinstance(style, str): style = gs_catalog.get_style(instance.name, workspace=instance.workspace) self.assertIsNotNone(style) self.assertFalse(isinstance(style, str)) instance.default_style, _ = Style.objects.get_or_create( name=style.name, defaults=dict(sld_title=style.sld_title, sld_body=style.sld_body)) self.assertIsNotNone(instance.default_style) self.assertIsNotNone(instance.default_style.name) # WMS Links wms_links = wms_links(f"{ogc_settings.public_url}wms?", instance.alternate, bbox, srid, height, width) self.assertIsNotNone(wms_links) self.assertEqual(len(wms_links), 3) wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms') identifier = urlencode({'layers': instance.alternate}) for _link in wms_links: logger.debug(f'{wms_url} --> {_link[3]}') self.assertTrue(wms_url in _link[3]) logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) # WFS Links wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?", instance.alternate, bbox, srid) self.assertIsNotNone(wfs_links) self.assertEqual(len(wfs_links), 6) wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs') identifier = urlencode({'typename': instance.alternate}) for _link in wfs_links: logger.debug(f'{wfs_url} --> {_link[3]}') self.assertTrue(wfs_url in _link[3]) logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) # WCS Links wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?", instance.alternate, bbox, srid) self.assertIsNotNone(wcs_links) self.assertEqual(len(wcs_links), 2) wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs') identifier = urlencode( {'coverageid': instance.alternate.replace(':', '__', 1)}) for _link in wcs_links: logger.debug(f'{wcs_url} --> {_link[3]}') self.assertTrue(wcs_url in _link[3]) logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3])
from ..maps.models import Map from ..layers.models import Dataset from ..documents.models import Document from ..documents.enumerations import ( DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP) from ..people.utils import get_valid_user from ..layers.utils import resolve_regions from ..layers.metadata import convert_keyword from ..services.models import Service from ..harvesting.models import HarvestableResource logger = logging.getLogger(__name__) ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)['default'] class KeywordHandler: ''' Object needed to handle the keywords coming from the XML The expected input are: - instance (Dataset/Document/Map): instance of any object inherited from ResourceBase. - keywords (list(dict)): Is required to analyze the keywords to find if some thesaurus is available. ''' def __init__(self, instance, keywords): self.instance = instance self.keywords = keywords def set_keywords(self):
def get_map( ogc_server_location: str, layers: List, bbox: List, wms_version: str = settings.OGC_SERVER["default"].get("WMS_VERSION", "1.1.1"), mime_type: str = "image/png", styles: List = None, width: int = 240, height: int = 200, max_retries: int = 3, retry_delay: int = 1, ): """ Function fetching an image from OGC server. For the requests to the configured OGC backend (ogc_server_settings.LOCATION) the function tries to generate an access_token and attach it to the URL. If access_token is not added ant the request is against Geoserver Basic Authentication is used instead. If image retrieval fails, function retries to fetch the image max_retries times, waiting retry_delay seconds between consecutive requests. :param ogc_server_location: OGC server URL :param layers: layers which should be fetched from the OGC server :param bbox: area's bounding box in format: [west, east, south, north, CRS] :param wms_version: WMS version of the query (default: 1.1.1) :param mime_type: mime type of the returned image :param styles: styles, which OGC server should use for rendering an image :param width: width of the returned image :param height: height of the returned image :param max_retries: maximum number of retries before skipping retrieval :param retry_delay: number of seconds waited between retries :returns: retrieved image """ ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] if ogc_server_location is not None: thumbnail_url = ogc_server_location else: thumbnail_url = ogc_server_settings.LOCATION if thumbnail_url.startswith(ogc_server_settings.PUBLIC_LOCATION): thumbnail_url = thumbnail_url.replace(ogc_server_settings.PUBLIC_LOCATION, ogc_server_settings.LOCATION) wms_endpoint = "" additional_kwargs = {} if thumbnail_url == ogc_server_settings.LOCATION: # add access token to requests to Geoserver (logic based on the previous implementation) username = ogc_server_settings.credentials.username user = get_user_model().objects.filter(username=username).first() if user: access_token = get_or_create_token(user) if access_token and not access_token.is_expired(): additional_kwargs['access_token'] = access_token.token # add WMS endpoint to requests to Geoserver wms_endpoint = getattr(ogc_server_settings, "WMS_ENDPOINT") or "ows" # prepare authorization for WMS service headers = {} if thumbnail_url.startswith(ogc_server_settings.LOCATION): if "access_token" not in additional_kwargs.keys(): # for the Geoserver backend, use Basic Auth, if access_token is not provided _user, _pwd = ogc_server_settings.credentials encoded_credentials = base64.b64encode(f"{_user}:{_pwd}".encode("UTF-8")).decode("ascii") headers["Authorization"] = f"Basic {encoded_credentials}" else: headers["Authorization"] = f"Bearer {additional_kwargs['access_token']}" wms = WebMapService( f"{thumbnail_url}{wms_endpoint}", version=wms_version, headers=headers) image = None for retry in range(max_retries): try: # fetch data image = wms.getmap( layers=layers, styles=styles, srs=bbox[-1] if bbox else None, bbox=[bbox[0], bbox[2], bbox[1], bbox[3]] if bbox else None, size=(width, height), format=mime_type, transparent=True, timeout=getattr(ogc_server_settings, "TIMEOUT", None), **additional_kwargs, ) # validate response if not image or "ServiceException" in str(image.read()): raise ThumbnailError( f"Fetching partial thumbnail from {thumbnail_url} failed with response: {str(image)}" ) except Exception as e: if retry + 1 >= max_retries: logger.exception(e) return time.sleep(retry_delay) continue else: break return image.read()
def _datasets_locations( instance: Union[Dataset, Map], compute_bbox: bool = False, target_crs: str = "EPSG:3857") -> Tuple[List[List], List]: """ Function returning a list mapping instance's datasets to their locations, enabling to construct a minimum number of WMS request for multiple datasets of the same OGC source (ensuring datasets order for Maps) :param instance: instance of Dataset or Map models :param compute_bbox: flag determining whether a BBOX containing the instance should be computed, based on instance's datasets :param target_crs: valid only when compute_bbox is True - CRS of the returned BBOX :return: a tuple with a list, which maps datasets to their locations in a correct datasets order e.g. [ ["http://localhost:8080/geoserver/": ["geonode:layer1", "geonode:layer2]] ] and a list optionally consisting of 5 elements containing west, east, south, north instance's boundaries and CRS """ ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] locations = [] bbox = [] if isinstance(instance, Dataset): locations.append([ instance.ows_url or ogc_server_settings.LOCATION, [instance.alternate], [] ]) if compute_bbox: if instance.ll_bbox_polygon: bbox = utils.clean_bbox(instance.ll_bbox, target_crs) elif (instance.bbox[-1].upper() != 'EPSG:3857' and target_crs.upper() == 'EPSG:3857' and utils.exceeds_epsg3857_area_of_use(instance.bbox)): # handle exceeding the area of use of the default thumb's CRS bbox = utils.transform_bbox( utils.crop_to_3857_area_of_use(instance.bbox), target_crs) else: bbox = utils.transform_bbox(instance.bbox, target_crs) elif isinstance(instance, Map): for map_dataset in instance.maplayers.iterator(): if not map_dataset.local and not map_dataset.ows_url: logger.warning( "Incorrectly defined remote dataset encountered (no OWS URL defined)." "Skipping it in the thumbnail generation.") continue name = get_dataset_name(map_dataset) store = map_dataset.store workspace = get_dataset_workspace(map_dataset) map_dataset_style = map_dataset.current_style if store and Dataset.objects.filter( store=store, workspace=workspace, name=name).count() > 0: dataset = Dataset.objects.filter(store=store, workspace=workspace, name=name).first() elif workspace and Dataset.objects.filter(workspace=workspace, name=name).count() > 0: dataset = Dataset.objects.filter(workspace=workspace, name=name).first() elif Dataset.objects.filter( alternate=map_dataset.name).count() > 0: dataset = Dataset.objects.filter( alternate=map_dataset.name).first() else: logger.warning( f"Dataset for MapLayer {name} was not found. Skipping it in the thumbnail." ) continue if dataset.subtype in ['tileStore', 'remote']: # limit number of locations, ensuring dataset order if len(locations) and locations[-1][ 0] == dataset.remote_service.service_url: # if previous dataset's location is the same as the current one - append current dataset there locations[-1][1].append(dataset.alternate) # update the styles too if map_dataset_style: locations[-1][2].append(map_dataset_style) else: locations.append([ dataset.remote_service.service_url, [dataset.alternate], [map_dataset_style] if map_dataset_style else [] ]) else: # limit number of locations, ensuring dataset order if len(locations) and locations[-1][0] == settings.OGC_SERVER[ "default"]["LOCATION"]: # if previous dataset's location is the same as the current one - append current dataset there locations[-1][1].append(dataset.alternate) # update the styles too if map_dataset_style: locations[-1][2].append(map_dataset_style) else: locations.append([ settings.OGC_SERVER["default"]["LOCATION"], [dataset.alternate], [map_dataset_style] if map_dataset_style else [] ]) if compute_bbox: if dataset.ll_bbox_polygon: dataset_bbox = utils.clean_bbox(dataset.ll_bbox, target_crs) elif (dataset.bbox[-1].upper() != 'EPSG:3857' and target_crs.upper() == 'EPSG:3857' and utils.exceeds_epsg3857_area_of_use(dataset.bbox)): # handle exceeding the area of use of the default thumb's CRS dataset_bbox = utils.transform_bbox( utils.crop_to_3857_area_of_use(dataset.bbox), target_crs) else: dataset_bbox = utils.transform_bbox( dataset.bbox, target_crs) if not bbox: bbox = dataset_bbox else: # dataset's BBOX: (left, right, bottom, top) bbox = [ min(bbox[0], dataset_bbox[0]), max(bbox[1], dataset_bbox[1]), min(bbox[2], dataset_bbox[2]), max(bbox[3], dataset_bbox[3]), ] if bbox and len(bbox) < 5: bbox = list(bbox) + [target_crs] # convert bbox to list, if it's tuple return locations, bbox