def layer_batch_download(request): """ batch download a set of layers POST - begin download GET?id=<download_id> monitor status """ from geonode.utils import http_client, _get_basic_auth_info # currently this just piggy-backs on the map download backend # by specifying an ad hoc map that contains all layers requested # for download. assumes all layers are hosted locally. # status monitoring is handled slightly differently. if request.method == 'POST': layers = request.POST.getlist("layer") layers = Layer.objects.filter(typename__in=list(layers)) def layer_son(layer): return { "name": layer.typename, "service": layer.service_type, "metadataURL": "", "serviceURL": "" } readme = """This data is provided by GeoNode.\n\nContents:""" def list_item(lyr): return "%s - %s.*" % (lyr.title, lyr.name) readme = "\n".join([readme] + [list_item(l) for l in layers]) fake_map = { "map": { "readme": readme }, "layers": [layer_son(lyr) for lyr in layers] } url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION resp, content = http_client.request(url, 'POST', body=json.dumps(fake_map)) return HttpResponse(content, status=resp.status) if request.method == 'GET': # essentially, this just proxies back to geoserver download_id = request.GET.get('id', None) if download_id is None: return HttpResponse(status=404) url = "%srest/process/batchDownload/status/%s" % ( ogc_server_settings.LOCATION, download_id) resp, content = http_client.request(url, 'GET') return HttpResponse(content, status=resp.status)
def layer_batch_download(request): """ batch download a set of layers POST - begin download GET?id=<download_id> monitor status """ from geonode.utils import http_client # currently this just piggy-backs on the map download backend # by specifying an ad hoc map that contains all layers requested # for download. assumes all layers are hosted locally. # status monitoring is handled slightly differently. if request.method == 'POST': layers = request.POST.getlist("layer") layers = Layer.objects.filter(typename__in=list(layers)) def layer_son(layer): return { "name": layer.typename, "service": layer.service_type, "metadataURL": "", "serviceURL": "" } readme = """This data is provided by GeoNode.\n\nContents:""" def list_item(lyr): return "%s - %s.*" % (lyr.title, lyr.name) readme = "\n".join([readme] + [list_item(l) for l in layers]) fake_map = { "map": {"readme": readme}, "layers": [layer_son(lyr) for lyr in layers] } url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION resp, content = http_client.request( url, 'POST', body=json.dumps(fake_map)) return HttpResponse(content, status=resp.status) if request.method == 'GET': # essentially, this just proxies back to geoserver download_id = request.GET.get('id', None) if download_id is None: return HttpResponse(status=404) url = "%srest/process/batchDownload/status/%s" % ( ogc_server_settings.LOCATION, download_id) resp, content = http_client.request(url, 'GET') return HttpResponse(content, status=resp.status)
def probe_service(self): from geonode.utils import http_client try: resp, content = http_client.request(self.service_url) return resp.status_code except Exception: return 404
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ mapObject = _resolve_map(request, mapid, 'maps.view_map') map_status = dict() if request.method == 'POST': url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION def perm_filter(layer): return request.user.has_perm('layers.view_layer', obj=layer) mapJson = mapObject.json(perm_filter) # we need to remove duplicate layers j_map = json.loads(mapJson) j_layers = j_map["layers"] for j_layer in j_layers: if(len([l for l in j_layers if l == j_layer]))>1: j_layers.remove(j_layer) mapJson = json.dumps(j_map) resp, content = http_client.request(url, 'POST', body=mapJson) status = int(resp.status) if status == 200: map_status = json.loads(content) request.session["map_status"] = map_status else: raise Exception('Could not start the download of %s. Error was: %s' % (mapObject.title, content)) locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in mapObject.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(typename=lyr.name) if not request.user.has_perm('layers.view_layer', obj=ownable_layer): locked_layers.append(lyr) else: # we need to add the layer only once if len([l for l in downloadable_layers if l.name == lyr.name]) == 0: downloadable_layers.append(lyr) return render_to_response(template, RequestContext(request, { "map_status" : map_status, "map" : mapObject, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "site" : settings.SITEURL }))
def _fetch_thumb_and_compare(self, url, expected_image): if url == missing_thumbnail_url: logger.error(f'It was not possible to fetch the remote layer WMS GetMap! thumb_url: {url}') return _, img = http_client.request(url) content = BytesIO(img) Image.open(content).verify() # verify that it is, in fact an image thumb = Image.open(content) diff = Image.new("RGB", thumb.size) mismatch = pixelmatch(thumb, expected_image, diff) if mismatch >= expected_image.size[0] * expected_image.size[1] * 0.01: logger.warn("Mismatch, it was not possible to bump the bg!") # Sometimes this test fails to fetch the OSM background with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.png', delete=False) as tmpfile: logger.error(f"Dumping thumb to: {tmpfile.name}") thumb.save(tmpfile) # Let's check that the thumb is valid at least with Image.open(tmpfile) as img: img.verify() with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.png', delete=False) as tmpfile: logger.error(f"Dumping diff to: {tmpfile.name}") diff.save(tmpfile) # Let's check that the thumb is valid at least with Image.open(tmpfile) as img: img.verify() else: self.assertTrue( mismatch < expected_image.size[0] * expected_image.size[1] * 0.01, "Expected test and pre-generated thumbnails to differ up to 1%", )
def probe(self): from geonode.utils import http_client try: resp, _ = http_client.request(self.url) return resp.status_code in (200, 201) except Exception: return False
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ mapObject = _resolve_map(request, mapid, 'maps.view_map') map_status = dict() if request.method == 'POST': url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION def perm_filter(layer): return request.user.has_perm('layers.view_layer', obj=layer) mapJson = mapObject.json(perm_filter) # we need to remove duplicate layers j_map = json.loads(mapJson) j_layers = j_map["layers"] for j_layer in j_layers: if(len([l for l in j_layers if l == j_layer]))>1: j_layers.remove(j_layer) mapJson = json.dumps(j_map) resp, content = http_client.request(url, 'POST', body=mapJson) if resp.status not in (400, 404, 417): map_status = json.loads(content) request.session["map_status"] = map_status else: pass # XXX fix locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in mapObject.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(typename=lyr.name) if not request.user.has_perm('layers.view_layer', obj=ownable_layer): locked_layers.append(lyr) else: # we need to add the layer only once if len([l for l in downloadable_layers if l.name == lyr.name]) == 0: downloadable_layers.append(lyr) return render_to_response(template, RequestContext(request, { "map_status" : map_status, "map" : mapObject, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "geoserver" : ogc_server_settings.public_url, "site" : settings.SITEURL }))
def _fetch_wp_json(): base_url = settings.BLOG_BASE_URL wp_json_url = base_url + "/wp-json/wp/v2/posts?_embed&page=1&per_page=3" response, content = http_client.request(wp_json_url, method='GET') _output_json = json.loads(content) for _item in _output_json: _item['image_url'] = _item['_embedded']['wp:featuredmedia'][0][ 'source_url'] return {'wp_posts': _output_json}
def create_thumbnail(instance, thumbnail_remote_url, thumbail_create_url=None): BBOX_DIFFERENCE_THRESHOLD = 1e-5 if not thumbail_create_url: thumbail_create_url = thumbnail_remote_url # Check if the bbox is invalid valid_x = (float(instance.bbox_x0) - float(instance.bbox_x1))**2 > BBOX_DIFFERENCE_THRESHOLD valid_y = (float(instance.bbox_y1) - float(instance.bbox_y0))**2 > BBOX_DIFFERENCE_THRESHOLD image = None if valid_x and valid_y: Link.objects.get_or_create(resource=instance.get_self_resource(), url=thumbnail_remote_url, defaults=dict( extension='png', name=_("Remote Thumbnail"), mime='image/png', link_type='image', )) # Download thumbnail and save it locally. resp, image = http_client.request(thumbail_create_url) if 'ServiceException' in image or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: # first delete thumbnail file on disk to prevent duplicates: if instance.has_thumbnail(): instance.thumbnail_set.get().thumb_file.delete() # update database and save new thumbnail file on disk: instance.thumbnail_set.all().delete() thumbnail = Thumbnail(thumb_spec=thumbnail_remote_url) instance.thumbnail_set.add(thumbnail) thumbnail.thumb_file.save('layer-%s-thumb.png' % instance.id, ContentFile(image)) thumbnail.save() thumbnail_url = urljoin(settings.SITEURL, instance.thumbnail_set.get().thumb_file.url) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_url, defaults=dict( name=_('Thumbnail'), extension='png', mime='image/png', link_type='image', )) ResourceBase.objects.filter(id=instance.id).update( thumbnail_url=instance.get_thumbnail_url())
def geoserver_post_save_map(instance, sender, **kwargs): local_layers = [] for layer in instance.layers: if layer.local: local_layers.append( Layer.objects.get(typename=layer.name).typename) params = { 'layers': ",".join(local_layers).encode('utf-8'), 'format': 'image/png8', 'width': 200, 'height': 150, } # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_url = ogc_server_settings.LOCATION + "wms/reflect?" + p Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_url, defaults=dict( extension='png', name=_("Remote Thumbnail"), mime='image/png', link_type='image', )) # Download thumbnail and save it locally. resp, image = http_client.request(thumbnail_url) if 'ServiceException' in image or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: #Clean any orphan Thumbnail before Thumbnail.objects.filter(resourcebase__id=None).delete() thumbnail, created = Thumbnail.objects.get_or_create( resourcebase__id=instance.id) thumbnail.thumb_spec = thumbnail_url thumbnail.save_thumb(image, instance._thumbnail_path()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=settings.SITEURL + instance._thumbnail_path(), defaults=dict( extension='png', name=_("Thumbnail"), mime='image/png', link_type='image', ))
def geoserver_post_save_map(instance, sender, **kwargs): instance.set_missing_info() local_layers = [] for layer in instance.layers: if layer.local: local_layers.append(Layer.objects.get(typename=layer.name).typename) image = None # If the map does not have any local layers, do not create the thumbnail. if len(local_layers) > 0: params = {"layers": ",".join(local_layers).encode("utf-8"), "format": "image/png8", "width": 200, "height": 150} # Add the bbox param only if the bbox is different to [None, None, None, None] if None not in instance.bbox: params["bbox"] = instance.bbox_string # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_remote_url = ogc_server_settings.LOCATION + "wms/reflect?" + p Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=thumbnail_remote_url, defaults=dict(extension="png", name=_("Remote Thumbnail"), mime="image/png", link_type="image"), ) # Download thumbnail and save it locally. resp, image = http_client.request(thumbnail_remote_url) if "ServiceException" in image or resp.status < 200 or resp.status > 299: msg = "Unable to obtain thumbnail: %s" % image logger.debug(msg) # Replace error message with None. image = None if image is not None: if instance.has_thumbnail(): instance.thumbnail.thumb_file.delete() else: instance.thumbnail = Thumbnail() instance.thumbnail.thumb_file.save("map-%s-thumb.png" % instance.id, ContentFile(image)) instance.thumbnail.thumb_spec = thumbnail_remote_url instance.thumbnail.save() thumbnail_url = urljoin(settings.SITEURL, instance.thumbnail.thumb_file.url) Link.objects.get_or_create( resource=instance.resourcebase_ptr, url=thumbnail_url, defaults=dict(name=_("Thumbnail"), extension="png", mime="image/png", link_type="image"), )
def fetch_wms(url: str, max_retries: int = 3, retry_delay: int = 1): """ Function fetching an image from OGC server. The request is performed based on the WMS URL. In case access_token in not present in the URL , and Geoserver is used and the OGC backend, Basic Authentication is used instead. If image retrieval fails, function retries to fetch the image max_retries times, waiting retry_delay seconds between consecutive requests. :param url: WMS URL of the image :param max_retries: maximum number of retries before skipping retrieval :param retry_delay: number of seconds waited between retries :returns: retrieved image """ # prepare authorization for WMS service headers = {} if "access_token" not in url: if url.startswith(settings.OGC_SERVER["default"]["LOCATION"]): # for the Geoserver backend, use Basic Auth, if access_token is not provided _user = settings.OGC_SERVER["default"].get("USER") _pwd = settings.OGC_SERVER["default"].get("PASSWORD") encoded_credentials = base64.b64encode( f"{_user}:{_pwd}".encode("UTF-8")).decode("ascii") headers["Authorization"] = f"Basic {encoded_credentials}" image = None for retry in range(max_retries): try: # fetch data resp, image = http_client.request( url, headers=headers, timeout=settings.OGC_SERVER["default"].get("TIMEOUT", 60)) # validate response if not resp or resp.status_code < 200 or resp.status_code > 299 or "ServiceException" in str( image): _status_code = resp.status_code if resp else 'Unknown' logger.debug( f"Fetching partial thumbnail from {url} failed with status code: " f"{_status_code} and response: {str(image)}") image = None time.sleep(retry_delay) continue except Exception as e: if retry + 1 >= max_retries: logger.exception(e) time.sleep(retry_delay) continue else: break return image
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ mapObject = _resolve_map(request, mapid, 'maps.view_map') map_status = dict() if request.method == 'POST': url = "%srest/process/batchDownload/launch/" % settings.GEOSERVER_BASE_URL def perm_filter(layer): return request.user.has_perm('maps.view_layer', obj=layer) mapJson = mapObject.json(perm_filter) resp, content = http_client.request(url, 'POST', body=mapJson) if resp.status not in (400, 404, 417): map_status = json.loads(content) request.session["map_status"] = map_status else: pass # XXX fix locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in mapObject.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(typename=lyr.name) if not request.user.has_perm('maps.view_layer', obj=ownable_layer): locked_layers.append(lyr) else: downloadable_layers.append(lyr) return render_to_response( template, RequestContext( request, { "map_status": map_status, "map": mapObject, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "geoserver": settings.GEOSERVER_BASE_URL, "site": settings.SITEURL }))
def get_gs_thumbnail(instance): from geonode.geoserver.helpers import ogc_server_settings if instance.class_name == 'Map': local_layers = [] for layer in instance.layers: if layer.local: local_layers.append(layer.name) layers = ",".join(local_layers).encode('utf-8') if (len(local_layers) == 0): return None else: layers = instance.typename.encode('utf-8') params = { 'layers': layers, 'format': 'image/png8', 'width': 200, 'height': 150, 'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z' } # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_create_url = ogc_server_settings.LOCATION + \ "wms/reflect?" + p tries = 0 max_tries = 30 while tries < max_tries: logger.debug( 'Thumbnail: Requesting thumbnail from GeoServer. ' 'Attempt %d of %d', tries + 1, max_tries) resp, image = http_client.request(thumbnail_create_url) if 200 <= resp.status <= 299: if 'ServiceException' not in image: return image else: # Unexpected Error Code, Stop Trying logger.debug( 'Thumbnail: Encountered unexpected status code: %d. ' 'Aborting.', resp.status) logger.debug(resp) break # Layer not ready yet, try again tries += 1 time.sleep(1) return None
def create_thumbnail(instance, thumbnail_remote_url): BBOX_DIFFERENCE_THRESHOLD = 1e-5 #Check if the bbox is invalid valid_x = (float(instance.bbox_x0) - float(instance.bbox_x1))**2 > BBOX_DIFFERENCE_THRESHOLD valid_y = (float(instance.bbox_y1) - float(instance.bbox_y0))**2 > BBOX_DIFFERENCE_THRESHOLD image = None if valid_x and valid_y: Link.objects.get_or_create(resource= instance.get_self_resource(), url=thumbnail_remote_url, defaults=dict( extension='png', name=_("Remote Thumbnail"), mime='image/png', link_type='image', ) ) # Download thumbnail and save it locally. resp, image = http_client.request(thumbnail_remote_url) if 'ServiceException' in image or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: if instance.has_thumbnail(): instance.thumbnail.thumb_file.delete() else: instance.thumbnail = Thumbnail() instance.thumbnail.thumb_file.save('layer-%s-thumb.png' % instance.id, ContentFile(image)) instance.thumbnail.thumb_spec = thumbnail_remote_url instance.thumbnail.save() thumbnail_url = urljoin(settings.SITEURL, instance.thumbnail.thumb_file.url) Link.objects.get_or_create(resource= instance.resourcebase_ptr, url=thumbnail_url, defaults=dict( name=_('Thumbnail'), extension='png', mime='image/png', link_type='image', ) ) ResourceBase.objects.filter(id=instance.id).update( thumbnail_url=instance.get_thumbnail_url() )
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ mapObject = _resolve_map(request, mapid, 'maps.view_map') map_status = dict() if request.method == 'POST': url = "%srest/process/batchDownload/launch/" % settings.GEOSERVER_BASE_URL def perm_filter(layer): return request.user.has_perm('maps.view_layer', obj=layer) mapJson = mapObject.json(perm_filter) resp, content = http_client.request(url, 'POST', body=mapJson) if resp.status not in (400, 404, 417): map_status = json.loads(content) request.session["map_status"] = map_status else: pass # XXX fix locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in mapObject.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(typename=lyr.name) if not request.user.has_perm('maps.view_layer', obj=ownable_layer): locked_layers.append(lyr) else: downloadable_layers.append(lyr) return render_to_response(template, RequestContext(request, { "map_status" : map_status, "map" : mapObject, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "geoserver" : settings.GEOSERVER_BASE_URL, "site" : settings.SITEURL }))
def test_map_default_thumb(self): create_gs_thumbnail_geonode(self.map_composition, overwrite=True) if not self.map_composition.has_thumbnail(): logger.warn("It was not possible to dump the background!") logger.error( f"map_composition thumb: {self.map_composition.thumbnail_url}") else: _, img = http_client.request(self.map_composition.thumbnail_url) content = BytesIO(img) Image.open(content).verify() # verify that it is, in fact an image thumb = Image.open(content) diff = Image.new("RGB", thumb.size) expected_thumb = Image.open(EXPECTED_RESULTS_DIR + "thumbnails/default_map_thumb.png") mismatch = pixelmatch(thumb, expected_thumb, diff) if mismatch >= expected_thumb.size[0] * expected_thumb.size[ 1] * 0.01: logger.warn("Mismatch, it was not possible to bump the bg!") # Sometimes this test fails to fetch the OSM background with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.png', delete=False) as tmpfile: logger.error(f"Dumping thumb to: {tmpfile.name}") thumb.save(tmpfile) # Let's check that the thumb is valid at least with Image.open(tmpfile) as img: img.verify() with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.png', delete=False) as tmpfile: logger.error(f"Dumping diff to: {tmpfile.name}") diff.save(tmpfile) # Let's check that the thumb is valid at least with Image.open(tmpfile) as img: img.verify() else: self.assertTrue( mismatch < expected_thumb.size[0] * expected_thumb.size[1] * 0.01, "Expected test and pre-generated thumbnails to differ up to 1%", )
def map_download_check(request): """ this is an endpoint for monitoring map downloads """ try: layer = request.session["map_status"] if type(layer) == dict: url = "%srest/process/batchDownload/status/%s" % (ogc_server_settings.LOCATION,layer["id"]) resp,content = http_client.request(url,'GET') status= resp.status if resp.status == 400: return HttpResponse(content="Something went wrong",status=status) else: content = "Something Went wrong" status = 400 except ValueError: # TODO: Is there any useful context we could include in this log? logger.warn("User tried to check status, but has no download in progress.") return HttpResponse(content=content,status=status)
def get_gs_thumbnail(instance): from geonode.geoserver.helpers import ogc_server_settings if instance.class_name == 'Map': local_layers = [] for layer in instance.layers: if layer.local: local_layers.append(layer.name) layers = ",".join(local_layers).encode('utf-8') if (len(local_layers) == 0): return None else: layers = instance.typename.encode('utf-8') params = { 'layers': layers, 'format': 'image/png8', 'width': 200, 'height': 150, 'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z' } # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_create_url = ogc_server_settings.LOCATION + \ "wms/reflect?" + p resp, image = http_client.request(thumbnail_create_url) if ('ServiceException' in image or resp.status < 200 or resp.status > 299): return None return image
def _enrich_layer_metadata(self, geonode_service, geonode_layer): workspace, layername = geonode_layer.name.split( ":") if ":" in geonode_layer.name else (None, geonode_layer.name) url = urlsplit(self.url) base_url = f'{url.scheme}://{url.netloc}/' response = requests.get(f'{base_url}api/layers/?name={layername}', {}, timeout=10, verify=False) content = response.content status = response.status_code content_type = response.headers['Content-Type'] if status == 200 and 'application/json' == content_type: try: if isinstance(content, bytes): content = content.decode('UTF-8') _json_obj = json.loads(content) if _json_obj['meta']['total_count'] == 1: _layer = _json_obj['objects'][0] if _layer: r_fields = {} # Update plain fields for field in GeoNodeServiceHandler.LAYER_FIELDS: if field in _layer and _layer[field]: r_fields[field] = _layer[field] if r_fields: Layer.objects.filter(id=geonode_layer.id).update( **r_fields) geonode_layer.refresh_from_db() # Update Thumbnail if "thumbnail_url" in _layer and _layer[ "thumbnail_url"]: thumbnail_remote_url = _layer["thumbnail_url"] _url = urlsplit(thumbnail_remote_url) if not _url.scheme: thumbnail_remote_url = f"{geonode_layer.remote_service.service_url}{_url.path}" resp, image = http_client.request( thumbnail_remote_url) if 'ServiceException' in str(image) or \ resp.status_code < 200 or resp.status_code > 299: msg = f'Unable to obtain thumbnail: {image}' logger.debug(msg) # Replace error message with None. image = None if image is not None: thumbnail_name = f'layer-{geonode_layer.uuid}-thumb.png' geonode_layer.save_thumbnail(thumbnail_name, image=image) else: self._create_layer_thumbnail(geonode_layer) else: self._create_layer_thumbnail(geonode_layer) # Add Keywords if "keywords" in _layer and _layer["keywords"]: keywords = _layer["keywords"] if keywords: geonode_layer.keywords.clear() geonode_layer.keywords.add(*keywords) # Add Regions if "regions" in _layer and _layer["regions"]: (regions_resolved, regions_unresolved) = resolve_regions( _layer["regions"]) if regions_resolved: geonode_layer.regions.clear() geonode_layer.regions.add(*regions_resolved) # Remove temporary topic category here as the field converted into m2m field # any idea how to setup? # Add Topic Category # if "category" in _layer and _layer["category"]: # (categories_resolved, categories_unresolved) = resolve_categories(_layer["category"]) # if categories_resolved: # geonode_layer.category.clear() # geonode_layer.category.add(*categories_resolved) # if "category__gn_description" in _layer and _layer["category__gn_description"]: # try: # categories = TopicCategory.objects.filter( # Q(gn_description__iexact=_layer["category__gn_description"])) # if categories: # geonode_layer.category = categories[0] # except Exception: # traceback.print_exc() except Exception: traceback.print_exc() finally: try: geonode_layer.save(notify=True) except Exception as e: logger.error(e)
def proxy(request, url=None, response_callback=None, sec_chk_hosts=True, sec_chk_rules=True, timeout=None, allowed_hosts=[], **kwargs): # Request default timeout if not timeout: timeout = TIMEOUT # Security rules and settings PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ()) # Sanity url checks if 'url' not in request.GET and not url: return HttpResponse( "The proxy service requires a URL-encoded URL as a parameter.", status=400, content_type="text/plain") raw_url = url or request.GET['url'] raw_url = urljoin(settings.SITEURL, raw_url) if raw_url.startswith("/") else raw_url url = urlsplit(raw_url) scheme = str(url.scheme) locator = str(url.path) if url.query != "": locator += '?' + url.query if url.fragment != "": locator += '#' + url.fragment # White-Black Listing Hosts site_url = urlsplit(settings.SITEURL) if sec_chk_hosts and not settings.DEBUG: # Attach current SITEURL if site_url.hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += (site_url.hostname, ) # Attach current hostname if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import ogc_server_settings hostname = ( ogc_server_settings.hostname, ) if ogc_server_settings else () if hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += hostname # Check OWS regexp if url.query and ows_regexp.match(url.query): ows_tokens = ows_regexp.match(url.query).groups() if len( ows_tokens ) == 4 and 'version' == ows_tokens[0] and StrictVersion( ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion( ows_tokens[1] ) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in ( 'getcapabilities') and ows_tokens[3].upper() in ( 'OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'): if url.hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += (url.hostname, ) # Check Remote Services base_urls from geonode.services.models import Service for _s in Service.objects.all(): _remote_host = urlsplit(_s.base_url).hostname PROXY_ALLOWED_HOSTS += (_remote_host, ) if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS): return HttpResponse( "DEBUG is set to False but the host of the path provided to the proxy service" " is not in the PROXY_ALLOWED_HOSTS setting.", status=403, content_type="text/plain") # Security checks based on rules; allow only specific requests if sec_chk_rules: # TODO: Not yet implemented pass # Collecting headers and cookies headers, access_token = get_headers(request, url, raw_url, allowed_hosts=allowed_hosts) # Inject access_token if necessary parsed = urlparse(raw_url) parsed._replace(path=locator.encode('utf8')) if parsed.netloc == site_url.netloc and scheme != site_url.scheme: parsed = parsed._replace(scheme=site_url.scheme) _url = parsed.geturl() # Some clients / JS libraries generate URLs with relative URL paths, e.g. # "http://host/path/path/../file.css", which the requests library cannot # currently handle (https://github.com/kennethreitz/requests/issues/2982). # We parse and normalise such URLs into absolute paths before attempting # to proxy the request. _url = URL.from_text(_url).normalize().to_text() if request.method == "GET" and access_token and 'access_token' not in _url: query_separator = '&' if '?' in _url else '?' _url = ('%s%saccess_token=%s' % (_url, query_separator, access_token)) _data = request.body.decode('utf-8') # Avoid translating local geoserver calls into external ones if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import ogc_server_settings _url = _url.replace('%s%s' % (settings.SITEURL, 'geoserver'), ogc_server_settings.LOCATION.rstrip('/')) _data = _data.replace('%s%s' % (settings.SITEURL, 'geoserver'), ogc_server_settings.LOCATION.rstrip('/')) response, content = http_client.request(_url, method=request.method, data=_data, headers=headers, timeout=timeout, user=request.user) content = response.content or response.reason status = response.status_code content_type = response.headers.get('Content-Type') if status >= 400: return HttpResponse(content=content, reason=content, status=status, content_type=content_type) # decompress GZipped responses if not enabled # if content and response and response.getheader('Content-Encoding') == 'gzip': if content and content_type and content_type == 'gzip': buf = io.BytesIO(content) f = gzip.GzipFile(fileobj=buf) content = f.read() PLAIN_CONTENT_TYPES = ['text', 'plain', 'html', 'json', 'xml', 'gml'] for _ct in PLAIN_CONTENT_TYPES: if content_type and _ct in content_type and not isinstance( content, six.string_types): try: content = content.decode() break except Exception: pass if response and response_callback: kwargs = {} if not kwargs else kwargs kwargs.update({ 'response': response, 'content': content, 'status': status, 'content_type': content_type }) return response_callback(**kwargs) else: # If we get a redirect, let's add a useful message. if status and status in (301, 302, 303, 307): _response = HttpResponse( ('This proxy does not support redirects. The server in "%s" ' 'asked for a redirect to "%s"' % (url, response.getheader('Location'))), status=status, content_type=content_type) _response['Location'] = response.getheader('Location') return _response else: def _get_message(text): _s = text if isinstance(text, bytes): _s = text.decode("utf-8", "replace") try: found = re.search('<b>Message</b>(.+?)</p>', _s).group(1).strip() except Exception: found = _s return found return HttpResponse( content=content, reason=_get_message(content) if status not in (200, 201) else None, status=status, content_type=content_type)
def _render_thumbnail(self, spec): resp, content = http_client.request(spec) if resp.status < 200 or resp.status > 299: logger.warning('Unable to obtain thumbnail: %s', content) return return content
def geoserver_post_save_map(instance, sender, **kwargs): instance.set_missing_info() local_layers = [] for layer in instance.layers: if layer.local: local_layers.append(layer.name) image = None # If the map does not have any local layers, do not create the thumbnail. if len(local_layers) > 0: params = { 'layers': ",".join(local_layers).encode('utf-8'), 'format': 'image/png8', 'width': 200, 'height': 150, } # Add the bbox param only if the bbox is different to [None, None, # None, None] if None not in instance.bbox: params['bbox'] = instance.bbox_string # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_remote_url = ogc_server_settings.LOCATION + \ "wms/reflect?" + p Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_remote_url, defaults=dict( extension='png', name=_("Remote Thumbnail"), mime='image/png', link_type='image', ) ) # Download thumbnail and save it locally. resp, image = http_client.request(thumbnail_remote_url) if 'ServiceException' in image or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: if instance.has_thumbnail(): instance.thumbnail_set.get().thumb_file.delete() else: instance.thumbnail_set.add(Thumbnail()) instance.thumbnail_set.get().thumb_file.save( 'map-%s-thumb.png' % instance.id, ContentFile(image)) instance.thumbnail_set.get().thumb_spec = thumbnail_remote_url instance.thumbnail_set.get().save() thumbnail_url = urljoin( settings.SITEURL, instance.thumbnail_set.get().thumb_file.url) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_url, defaults=dict( name=_('Thumbnail'), extension='png', mime='image/png', link_type='image', ) ) from geonode.maps.models import Map Map.objects.filter(id=instance.id).update(thumbnail_url=thumbnail_url)
def proxy(request, url=None, response_callback=None, sec_chk_hosts=True, sec_chk_rules=True, timeout=None, **kwargs): # Request default timeout if not timeout: timeout = TIMEOUT # Security rules and settings PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ()) # Sanity url checks if 'url' not in request.GET and not url: return HttpResponse("The proxy service requires a URL-encoded URL as a parameter.", status=400, content_type="text/plain" ) raw_url = url or request.GET['url'] raw_url = urljoin( settings.SITEURL, raw_url) if raw_url.startswith("/") else raw_url url = urlsplit(raw_url) locator = str(url.path) if url.query != "": locator += '?' + url.query if url.fragment != "": locator += '#' + url.fragment # White-Black Listing Hosts if sec_chk_hosts and not settings.DEBUG: site_url = urlsplit(settings.SITEURL) if site_url.hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += (site_url.hostname, ) if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import ogc_server_settings hostname = ( ogc_server_settings.hostname, ) if ogc_server_settings else () if hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += hostname if url.query and ows_regexp.match(url.query): ows_tokens = ows_regexp.match(url.query).groups() if len(ows_tokens) == 4 and 'version' == ows_tokens[0] and StrictVersion( ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion( ows_tokens[1]) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in ( 'getcapabilities') and ows_tokens[3].upper() in ('OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'): if url.hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += (url.hostname, ) if not validate_host( url.hostname, PROXY_ALLOWED_HOSTS): return HttpResponse("DEBUG is set to False but the host of the path provided to the proxy service" " is not in the PROXY_ALLOWED_HOSTS setting.", status=403, content_type="text/plain" ) # Security checks based on rules; allow only specific requests if sec_chk_rules: # TODO: Not yet implemented pass # Collecting headers and cookies headers, access_token = get_headers(request, url, raw_url) # Inject access_token if necessary parsed = urlparse(raw_url) parsed._replace(path=locator.encode('utf8')) _url = parsed.geturl() if request.method == "GET" and access_token and 'access_token' not in _url: query_separator = '&' if '?' in _url else '?' _url = ('%s%saccess_token=%s' % (_url, query_separator, access_token)) response, content = http_client.request(_url, method=request.method, data=request.body, headers=headers, timeout=timeout, user=request.user) content = response.content or response.reason status = response.status_code content_type = response.headers.get('Content-Type') # decompress GZipped responses if not enabled # if content and response and response.getheader('Content-Encoding') == 'gzip': if content and content_type and content_type == 'gzip': from StringIO import StringIO import gzip buf = StringIO(content) f = gzip.GzipFile(fileobj=buf) content = f.read() if response and response_callback: kwargs = {} if not kwargs else kwargs kwargs.update({ 'response': response, 'content': content, 'status': status, 'content_type': content_type }) return response_callback(**kwargs) else: # If we get a redirect, let's add a useful message. if status and status in (301, 302, 303, 307): _response = HttpResponse(('This proxy does not support redirects. The server in "%s" ' 'asked for a redirect to "%s"' % (url, response.getheader('Location'))), status=status, content_type=content_type ) _response['Location'] = response.getheader('Location') return _response else: def _get_message(text): _s = text.decode("utf-8", "replace") try: found = re.search('<b>Message</b>(.+?)</p>', _s).group(1).strip() except BaseException: found = _s return found return HttpResponse( content=content, reason=_get_message(content) if status not in (200, 201) else None, status=status, content_type=content_type)
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ if not mapid.isdigit(): map_obj = _resolve_map_custom(request, mapid, 'urlsuffix', 'base.view_resourcebase', _PERMISSION_MSG_VIEW) else: map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) map_status = dict() if request.method == 'POST': url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION def perm_filter(layer): return request.user.has_perm('base.view_resourcebase', obj=layer.get_self_resource()) mapJson = map_obj.json(perm_filter) # we need to remove duplicate layers j_map = json.loads(mapJson) j_layers = j_map["layers"] for j_layer in j_layers: if (len([l for l in j_layers if l == j_layer])) > 1: j_layers.remove(j_layer) mapJson = json.dumps(j_map) resp, content = http_client.request(url, 'POST', body=mapJson) status = int(resp.status) if status == 200: map_status = json.loads(content) request.session["map_status"] = map_status else: raise Exception( 'Could not start the download of %s. Error was: %s' % (map_obj.title, content)) locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in map_obj.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(typename=lyr.name) if not request.user.has_perm( 'view_resourcebase', obj=ownable_layer.get_self_resource()): locked_layers.append(lyr) else: # we need to add the layer only once if len( [l for l in downloadable_layers if l.name == lyr.name]) == 0: downloadable_layers.append(lyr) return render_to_response( template, RequestContext( request, { "map_status": map_status, "map": map_obj, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "site": settings.SITEURL }))
def _enrich_layer_metadata(self, geonode_layer): url = urlsplit(self.url) if url.scheme == 'https': conn = HTTPSConnection(url.hostname, url.port) else: conn = HTTPConnection(url.hostname, url.port) workspace, layername = geonode_layer.name.split( ":") if ":" in geonode_layer.name else (None, geonode_layer.name) conn.request('GET', '/api/layers/?name=%s' % layername, '', {}) response = conn.getresponse() content = response.read() status = response.status content_type = response.getheader("Content-Type", "text/plain") if status == 200 and 'application/json' == content_type: try: _json_obj = json.loads(content) if _json_obj['meta']['total_count'] == 1: _layer = _json_obj['objects'][0] if _layer: r_fields = {} # Update plain fields for field in GeoNodeServiceHandler.LAYER_FIELDS: if field in _layer and _layer[field]: r_fields[field] = _layer[field] if r_fields: Layer.objects.filter( id=geonode_layer.id).update( **r_fields) geonode_layer.refresh_from_db() # Update Thumbnail if "thumbnail_url" in _layer and _layer["thumbnail_url"]: thumbnail_remote_url = _layer["thumbnail_url"] _url = urlsplit(thumbnail_remote_url) if not _url.scheme: thumbnail_remote_url = "{}{}".format( geonode_layer.remote_service.service_url, _url.path) resp, image = http_client.request( thumbnail_remote_url) if 'ServiceException' in image or \ resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid geonode_layer.save_thumbnail( thumbnail_name, image=image) else: self._create_layer_thumbnail(geonode_layer) # Add Keywords if "keywords" in _layer and _layer["keywords"]: keywords = _layer["keywords"] if keywords: geonode_layer.keywords.clear() geonode_layer.keywords.add(*keywords) # Add Regions if "regions" in _layer and _layer["regions"]: (regions_resolved, regions_unresolved) = resolve_regions( _layer["regions"]) if regions_resolved: geonode_layer.regions.clear() geonode_layer.regions.add(*regions_resolved) # Add Topic Category if "category__gn_description" in _layer and _layer["category__gn_description"]: try: categories = TopicCategory.objects.filter( Q(gn_description__iexact=_layer["category__gn_description"])) if categories: geonode_layer.category = categories[0] except BaseException: traceback.print_exc() except BaseException: traceback.print_exc() finally: geonode_layer.save()
def training_download(request, id, template='trainings/training_download.html'): """ Download all the layers of a training as a batch """ training = get_object_or_404(Training, pk=id) training_status = dict() if request.method == 'POST': url = ("%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION) def perm_filter(layer): return request.user.has_perm('layers.view_layer', obj=layer) # here we build the json necessary to the rest batchDownload data = { 'layers': [], "map": { 'readme': training.abstract, 'title': training.title } } for layer in training.layers.all(): store_type = 'WFS' if layer.storeType == 'coverageStore': store_type = 'WCS' layer_data = { 'metadataURL': '', 'service': store_type, 'name': layer.typename, 'serviceURL': '' } data['layers'].append(layer_data) training_json = json.dumps(data) resp, content = http_client.request(url, 'POST', body=training_json) status = int(resp.status) if status == 200: training_status = json.loads(content) request.session["training_status"] = training_status else: raise Exception( 'Could not start the download of %s. Error was: %s' % (training.title, content)) downloadable_layers = [] for layer in training.layers.all(): if request.user.has_perm('layers.view_layer', obj=layer): downloadable_layers.append(layer) return render_to_response( template, RequestContext( request, { "training_status": training_status, "training": training, "downloadable_layers": downloadable_layers, "geoserver": ogc_server_settings.public_url, "site": settings.SITEURL }))
def map_download(request, mapid, template="maps/map_download.html"): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ try: map_obj = _resolve_map(request, mapid, "base.download_resourcebase", _PERMISSION_MSG_VIEW) except PermissionDenied: return HttpResponse(MSG_NOT_ALLOWED, status=403) except Exception: raise Http404(MSG_NOT_FOUND) if not map_obj: raise Http404(MSG_NOT_FOUND) map_status = dict() if request.method == "POST": def perm_filter(layer): return request.user.has_perm("base.view_resourcebase", obj=layer.get_self_resource()) mapJson = map_obj.json(perm_filter) # we need to remove duplicate layers j_map = json.loads(mapJson) j_datasets = j_map["layers"] for j_dataset in j_datasets: if j_dataset["service"] is None: j_datasets.remove(j_dataset) continue if (len([_l for _l in j_datasets if _l == j_dataset])) > 1: j_datasets.remove(j_dataset) mapJson = json.dumps(j_map) # the path to geoserver backend continue here url = urljoin(settings.SITEURL, reverse("download-map", kwargs={"mapid": mapid})) resp, content = http_client.request(url, "POST", data=mapJson) status = int(resp.status_code) if status == 200: map_status = json.loads(content) request.session["map_status"] = map_status else: raise Exception( f"Could not start the download of {map_obj.title}. Error was: {content}" ) locked_datasets = [] remote_datasets = [] downloadable_datasets = [] for lyr in map_obj.maplayers.iterator(): if lyr.group != "background": if not lyr.local: remote_datasets.append(lyr) else: ownable_dataset = Dataset.objects.get(alternate=lyr.name) if not request.user.has_perm( "download_resourcebase", obj=ownable_dataset.get_self_resource()): locked_datasets.append(lyr) else: # we need to add the layer only once if len([ _l for _l in downloadable_datasets if _l.name == lyr.name ]) == 0: downloadable_datasets.append(lyr) site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith( "http") else settings.SITEURL register_event(request, EventType.EVENT_DOWNLOAD, map_obj) return render( request, template, context={ "geoserver": ogc_server_settings.PUBLIC_LOCATION, "map_status": map_status, "map": map_obj, "locked_datasets": locked_datasets, "remote_datasets": remote_datasets, "downloadable_datasets": downloadable_datasets, "site": site_url, }, )
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ map_obj = _resolve_map( request, mapid, 'base.download_resourcebase', _PERMISSION_MSG_VIEW) map_status = dict() if request.method == 'POST': def perm_filter(layer): return request.user.has_perm( 'base.view_resourcebase', obj=layer.get_self_resource()) mapJson = map_obj.json(perm_filter) # we need to remove duplicate layers j_map = json.loads(mapJson) j_layers = j_map["layers"] for j_layer in j_layers: if j_layer["service"] is None: j_layers.remove(j_layer) continue if (len([_l for _l in j_layers if _l == j_layer])) > 1: j_layers.remove(j_layer) mapJson = json.dumps(j_map) if check_ogc_backend(geoserver.BACKEND_PACKAGE): # TODO the url needs to be verified on geoserver url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION elif check_ogc_backend(qgis_server.BACKEND_PACKAGE): url = urljoin(settings.SITEURL, reverse("qgis_server:download-map", kwargs={'mapid': mapid})) # qgis-server backend stop here, continue on qgis_server/views.py return redirect(url) # the path to geoserver backend continue here resp, content = http_client.request(url, 'POST', body=mapJson) status = int(resp.status_code) if status == 200: map_status = json.loads(content) request.session["map_status"] = map_status else: raise Exception( 'Could not start the download of %s. Error was: %s' % (map_obj.title, content)) locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in map_obj.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(alternate=lyr.name) if not request.user.has_perm( 'download_resourcebase', obj=ownable_layer.get_self_resource()): locked_layers.append(lyr) else: # we need to add the layer only once if len( [_l for _l in downloadable_layers if _l.name == lyr.name]) == 0: downloadable_layers.append(lyr) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL return render(request, template, context={ "geoserver": ogc_server_settings.PUBLIC_LOCATION, "map_status": map_status, "map": map_obj, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "site": site_url })
def map_download(request, mapid, template='maps/map_download.html'): """ Download all the layers of a map as a batch XXX To do, remove layer status once progress id done This should be fix because """ map_obj = _resolve_map(request, mapid, 'base.download_resourcebase', _PERMISSION_MSG_VIEW) map_status = dict() if request.method == 'POST': def perm_filter(layer): return request.user.has_perm('base.view_resourcebase', obj=layer.get_self_resource()) mapJson = map_obj.json(perm_filter) # we need to remove duplicate layers j_map = json.loads(mapJson) j_layers = j_map["layers"] for j_layer in j_layers: if j_layer["service"] is None: j_layers.remove(j_layer) continue if (len([_l for _l in j_layers if _l == j_layer])) > 1: j_layers.remove(j_layer) mapJson = json.dumps(j_map) if check_ogc_backend(qgis_server.BACKEND_PACKAGE): url = urljoin( settings.SITEURL, reverse("qgis_server:download-map", kwargs={'mapid': mapid})) # qgis-server backend stop here, continue on qgis_server/views.py return redirect(url) # the path to geoserver backend continue here resp, content = http_client.request(url, 'POST', body=mapJson) status = int(resp.status_code) if status == 200: map_status = json.loads(content) request.session["map_status"] = map_status else: raise Exception( 'Could not start the download of %s. Error was: %s' % (map_obj.title, content)) locked_layers = [] remote_layers = [] downloadable_layers = [] for lyr in map_obj.layer_set.all(): if lyr.group != "background": if not lyr.local: remote_layers.append(lyr) else: ownable_layer = Layer.objects.get(alternate=lyr.name) if not request.user.has_perm( 'download_resourcebase', obj=ownable_layer.get_self_resource()): locked_layers.append(lyr) else: # we need to add the layer only once if len([ _l for _l in downloadable_layers if _l.name == lyr.name ]) == 0: downloadable_layers.append(lyr) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith( 'http') else settings.SITEURL register_event(request, EventType.EVENT_DOWNLOAD, map_obj) return render(request, template, context={ "geoserver": ogc_server_settings.PUBLIC_LOCATION, "map_status": map_status, "map": map_obj, "locked_layers": locked_layers, "remote_layers": remote_layers, "downloadable_layers": downloadable_layers, "site": site_url })
def proxy(request, url=None, response_callback=None, sec_chk_hosts=True, sec_chk_rules=True, timeout=None, **kwargs): # Request default timeout if not timeout: timeout = TIMEOUT # Security rules and settings PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ()) # Sanity url checks if 'url' not in request.GET and not url: return HttpResponse( "The proxy service requires a URL-encoded URL as a parameter.", status=400, content_type="text/plain") raw_url = url or request.GET['url'] raw_url = urljoin(settings.SITEURL, raw_url) if raw_url.startswith("/") else raw_url url = urlsplit(raw_url) locator = str(url.path) if url.query != "": locator += '?' + url.query if url.fragment != "": locator += '#' + url.fragment # White-Black Listing Hosts if sec_chk_hosts and not settings.DEBUG: site_url = urlsplit(settings.SITEURL) if site_url.hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += (site_url.hostname, ) if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import ogc_server_settings hostname = ( ogc_server_settings.hostname, ) if ogc_server_settings else () if hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += hostname if url.query and ows_regexp.match(url.query): ows_tokens = ows_regexp.match(url.query).groups() if len( ows_tokens ) == 4 and 'version' == ows_tokens[0] and StrictVersion( ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion( ows_tokens[1] ) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in ( 'getcapabilities') and ows_tokens[3].upper() in ( 'OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'): if url.hostname not in PROXY_ALLOWED_HOSTS: PROXY_ALLOWED_HOSTS += (url.hostname, ) if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS): return HttpResponse( "DEBUG is set to False but the host of the path provided to the proxy service" " is not in the PROXY_ALLOWED_HOSTS setting.", status=403, content_type="text/plain") # Security checks based on rules; allow only specific requests if sec_chk_rules: # TODO: Not yet implemented pass # Collecting headers and cookies headers, access_token = get_headers(request, url, raw_url) # Inject access_token if necessary parsed = urlparse(raw_url) parsed._replace(path=locator.encode('utf8')) _url = parsed.geturl() if request.method == "GET" and access_token and 'access_token' not in _url: query_separator = '&' if '?' in _url else '?' _url = ('%s%saccess_token=%s' % (_url, query_separator, access_token)) response, content = http_client.request(_url, method=request.method, data=request.body, headers=headers, timeout=timeout, user=request.user) content = response.content status = response.status_code content_type = response.headers.get('Content-Type') # decompress GZipped responses if not enabled # if content and response and response.getheader('Content-Encoding') == 'gzip': if content and content_type and content_type == 'gzip': from StringIO import StringIO import gzip buf = StringIO(content) f = gzip.GzipFile(fileobj=buf) content = f.read() if response and response_callback: kwargs = {} if not kwargs else kwargs kwargs.update({ 'response': response, 'content': content, 'status': status, 'content_type': content_type }) return response_callback(**kwargs) else: # If we get a redirect, let's add a useful message. if status and status in (301, 302, 303, 307): _response = HttpResponse( ('This proxy does not support redirects. The server in "%s" ' 'asked for a redirect to "%s"' % (url, response.getheader('Location'))), status=status, content_type=content_type) _response['Location'] = response.getheader('Location') return _response else: return HttpResponse(content=content, status=status, content_type=content_type)
def geoserver_post_save_map(instance, sender, **kwargs): instance.set_missing_info() local_layers = [] for layer in instance.layers: if layer.local: local_layers.append( Layer.objects.get(typename=layer.name).typename) image = None # If the map does not have any local layers, do not create the thumbnail. if len(local_layers) > 0: params = { 'layers': ",".join(local_layers).encode('utf-8'), 'format': 'image/png8', 'width': 200, 'height': 150, } # Add the bbox param only if the bbox is different to [None, None, None, None] if None not in instance.bbox: params['bbox'] = instance.bbox_string # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_remote_url = ogc_server_settings.LOCATION + "wms/reflect?" + p Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_remote_url, defaults=dict( extension='png', name=_("Remote Thumbnail"), mime='image/png', link_type='image', )) # Download thumbnail and save it locally. resp, image = http_client.request(thumbnail_remote_url) if 'ServiceException' in image or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: if instance.has_thumbnail(): instance.thumbnail.thumb_file.delete() else: instance.thumbnail = Thumbnail() instance.thumbnail.thumb_file.save('map-%s-thumb.png' % instance.id, ContentFile(image)) instance.thumbnail.thumb_spec = thumbnail_remote_url instance.thumbnail.save() thumbnail_url = urljoin(settings.SITEURL, instance.thumbnail.thumb_file.url) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_url, defaults=dict( name=_('Thumbnail'), extension='png', mime='image/png', link_type='image', ))
def get_gs_thumbnail(instance): from geonode.geoserver.helpers import ogc_server_settings if instance.class_name == 'Map': local_layers = [] for layer in instance.layers: if layer.local: local_layers.append(layer.name) layers = ",".join(local_layers).encode('utf-8') if (len(local_layers) == 0): return None else: layers = instance.typename.encode('utf-8') logger.debug('Instance storeType: %s', instance.storeType) params = { 'layers': layers, 'format': 'image/png8', 'width': 200, 'height': 150, 'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z' } baseurl = ogc_server_settings.LOCATION + \ "wms/reflect?" if (instance.storeType == 'remoteStore'): params['request'] = 'getMap' params['service'] = 'wms' params['version'] = '1.3.0' params['bbox'] = '%s,%s,%s,%s' % (instance.bbox_x0, instance.bbox_y0, instance.bbox_x1, instance.bbox_y1) baseurl = instance.ows_url + '?' del params['TIME'] if all(s in baseurl.lower() for s in ['mapserver', 'wmsserver']): params['styles'] = '' params['crs'] = 'epsg:4326' # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_create_url = baseurl + p if (instance.storeType == 'remoteStore'): if (instance.service.type == 'REST'): thumbnail_create_url = "%s/info/thumbnail" % (instance.ows_url) tries = 0 max_tries = 15 while tries < max_tries: logger.debug( 'Thumbnail: Requesting thumbnail from GeoServer. ' 'Attempt %d of %d for %s', tries + 1, max_tries, thumbnail_create_url) if (instance.storeType == 'remoteStore'): if tries > 4: thumbnail_create_url = thumbnail_create_url.replace( 'image/png8', 'image/jpeg') resp, image = http_client.request(thumbnail_create_url) else: # Login using basic auth as geoserver admin user = settings.GEOSERVER_USER pword = settings.GEOSERVER_PASSWORD auth = base64.encodestring(user + ':' + pword) resp, image = http_client.request( thumbnail_create_url, 'GET', headers={'Authorization': 'Basic ' + auth}) if 200 <= resp.status <= 299: if 'ServiceException' not in image: return image else: logger.debug( 'Thumbnail: Encountered unexpected status code: %d. ' 'Aborting.', resp.status) logger.debug(resp) else: # Unexpected Error Code, Stop Trying logger.debug( 'Thumbnail: Encountered unexpected status code: %d. ' 'Aborting.', resp.status) logger.debug(resp) break # Layer not ready yet, try again tries += 1 time.sleep(3) return None
def _render_thumbnail(self, spec): resp, content = http_client.request(spec) if 'ServiceException' in content or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % content raise RuntimeError(msg) return content
def _enrich_layer_metadata(self, geonode_layer): workspace, layername = geonode_layer.name.split( ":") if ":" in geonode_layer.name else (None, geonode_layer.name) url = urlsplit(self.url) base_url = '%s://%s/' % (url.scheme, url.netloc) response = requests.get('%sapi/layers/?name=%s' % (base_url, layername), {}, timeout=10, verify=False) content = response.content status = response.status_code content_type = response.headers['Content-Type'] if status == 200 and 'application/json' == content_type: try: if isinstance(content, bytes): content = content.decode('UTF-8') _json_obj = json.loads(content) if _json_obj['meta']['total_count'] == 1: _layer = _json_obj['objects'][0] if _layer: r_fields = {} # Update plain fields for field in GeoNodeServiceHandler.LAYER_FIELDS: if field in _layer and _layer[field]: r_fields[field] = _layer[field] if r_fields: Layer.objects.filter(id=geonode_layer.id).update( **r_fields) geonode_layer.refresh_from_db() # Update Thumbnail if "thumbnail_url" in _layer and _layer[ "thumbnail_url"]: thumbnail_remote_url = _layer["thumbnail_url"] _url = urlsplit(thumbnail_remote_url) if not _url.scheme: thumbnail_remote_url = "{}{}".format( geonode_layer.remote_service.service_url, _url.path) resp, image = http_client.request( thumbnail_remote_url) if 'ServiceException' in str(image) or \ resp.status_code < 200 or resp.status_code > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid geonode_layer.save_thumbnail(thumbnail_name, image=image) else: self._create_layer_thumbnail(geonode_layer) else: self._create_layer_thumbnail(geonode_layer) # Add Keywords if "keywords" in _layer and _layer["keywords"]: keywords = _layer["keywords"] if keywords: geonode_layer.keywords.clear() geonode_layer.keywords.add(*keywords) # Add Regions if "regions" in _layer and _layer["regions"]: (regions_resolved, regions_unresolved) = resolve_regions( _layer["regions"]) if regions_resolved: geonode_layer.regions.clear() geonode_layer.regions.add(*regions_resolved) # Add Topic Category if "category__gn_description" in _layer and _layer[ "category__gn_description"]: try: categories = TopicCategory.objects.filter( Q(gn_description__iexact=_layer[ "category__gn_description"])) if categories: geonode_layer.category = categories[0] except Exception: traceback.print_exc() except Exception: traceback.print_exc() finally: try: geonode_layer.save(notify=True) except Exception as e: logger.error(e)
def fetch(self, bbox: typing.List, zoom: int = None, *args, **kwargs): """ The function fetching tiles from a Slippy Map provider, composing them into a single image, and cropping it to match the given BBOX. Retrieval of each tile is repeated self.max_retries times, waiting self.retry_delay seconds between consecutive requests. :param bbox: bounding box of the background image, dataset compliant format: [west, east, south, north, CRS] :param zoom: zoom with which to retrieve Slippy Map's tiles (by default, it's calculated based on width, height) :return: None if the CRS is different from self.tiles_crs, or background Image """ if not self.url: logger.error("Thumbnail background requires url to be configured.") raise ThumbnailError("Tiled background improperly configured.") if bbox[-1].lower() != self.crs.lower(): # background service is not available the requested CRS CRS logger.debug( f"Thumbnail background generation skipped. " f"Clashing CRSs: requested {bbox[-1]}, supported {self.crs}") return bbox = [float(coord) for coord in bbox[0:4]] # check if BBOX fits within the EPSG:3857 map, if not - return an empty background if bbox[2] > self._epsg3857_max_y or bbox[3] < -self._epsg3857_max_y: return Image.new("RGB", (self.thumbnail_width, self.thumbnail_height), (250, 250, 250)) bbox4326 = self.bbox3857to4326(*bbox) # change bbox from dataset (left, right, bottom, top) to mercantile (left, bottom, right, top) self._mercantile_bbox = [ bbox4326[0], bbox4326[2], bbox4326[1], bbox4326[3] ] # calculate zoom level if zoom is None: zoom = self.calculate_zoom() else: zoom = int(zoom) top_left_tile = mercantile.tile(bbox4326[0], bbox4326[3], zoom) bottom_right_tile = mercantile.tile(bbox4326[1], bbox4326[2], zoom) # rescaling factors - indicators of how west and east BBOX boundaries are offset in respect to the world's map; # east and west boundaries may exceed the maximum coordinate of the world in EPSG:3857. In such case additinal # number of tiles need to be fetched to compose the image and the boundary tiles' coordinates need to be # rescaled to ensure the proper image cropping. epsg3857_world_width = 2 * self._epsg3857_max_x west_rescaling_factor = 0 if abs(bbox[0]) > self._epsg3857_max_x: west_rescaling_factor = ceil( (abs(bbox[0]) - self._epsg3857_max_x) / epsg3857_world_width) * copysign(1, bbox[0]) east_rescaling_factor = 0 if abs(bbox[1]) > self._epsg3857_max_x: east_rescaling_factor = ceil( (abs(bbox[1]) - self._epsg3857_max_x) / epsg3857_world_width) * copysign(1, bbox[1]) map_row_tiles = 2**zoom - 1 # number of tiles in the Map's row for a certain zoom level map_worlds = int(east_rescaling_factor - west_rescaling_factor) # number maps in an image worlds_between = map_worlds - 1 # number of full maps in an image if top_left_tile.x > bottom_right_tile.x or bbox[1] - bbox[ 0] > epsg3857_world_width or map_worlds > 0: # BBOX crosses Slippy Map's border if worlds_between > 0: tiles_rows = (list(range(top_left_tile.x, map_row_tiles + 1)) + worlds_between * list(range(map_row_tiles + 1)) + list(range(bottom_right_tile.x + 1))) else: tiles_rows = list(range(top_left_tile.x, map_row_tiles + 1)) + list( range(bottom_right_tile.x + 1)) else: # BBOx is contained by the Slippy Map if worlds_between > 0: tiles_rows = list( range(top_left_tile.x, bottom_right_tile.x + 1)) + worlds_between * list(range(map_row_tiles + 1)) else: tiles_rows = list( range(top_left_tile.x, bottom_right_tile.x + 1)) tiles_cols = list(range(top_left_tile.y, bottom_right_tile.y + 1)) # if latitude boundaries extend world's height - add background's height, and set constant Y offset for tiles additional_height = 0 fixed_top_offset = 0 fixed_bottom_offset = 0 north_extension3857 = max(0, bbox[3] - self._epsg3857_max_y) south_extension3857 = abs(min(0, bbox[2] + self._epsg3857_max_y)) extension3857 = north_extension3857 + south_extension3857 if extension3857: # get single tile's height in ESPG:3857 tile_bounds = mercantile.bounds(tiles_rows[0], tiles_cols[0], zoom) _, south = self.point4326to3857(getattr(tile_bounds, "west"), getattr(tile_bounds, "south")) _, north = self.point4326to3857(getattr(tile_bounds, "west"), getattr(tile_bounds, "north")) tile_hight3857 = north - south additional_height = round( self.tile_size * extension3857 / tile_hight3857) # based on linear proportion if north_extension3857: fixed_top_offset = round(self.tile_size * north_extension3857 / tile_hight3857) if south_extension3857: fixed_bottom_offset = round( self.tile_size * south_extension3857 / tile_hight3857) background = Image.new( "RGB", (len(tiles_rows) * self.tile_size, len(tiles_cols) * self.tile_size + additional_height), (250, 250, 250), ) for offset_x, x in enumerate(tiles_rows): for offset_y, y in enumerate(tiles_cols): if self.tms: y = (2**zoom) - y - 1 imgurl = self.url.format(x=x, y=y, z=zoom) im = None for retries in range(self.max_retries): try: resp, content = http_client.request(imgurl) if resp.status_code > 400: retries = self.max_retries - 1 raise Exception(f"{strip_tags(content)}") im = BytesIO(content) Image.open( im).verify() # verify that it is, in fact an image break except Exception as e: logger.error( f"Thumbnail background fetching from {imgurl} failed {retries} time(s) with: {e}" ) if retries + 1 == self.max_retries: raise e time.sleep(self.retry_delay) continue if im: image = Image.open( im ) # "re-open" the file (required after running verify method) # add the fetched tile to the background image, placing it under proper coordinates background.paste( image, (offset_x * self.tile_size, offset_y * self.tile_size + fixed_top_offset)) # get BBOX of the tiles top_left_bounds = mercantile.bounds(top_left_tile) bottom_right_bounds = mercantile.bounds(bottom_right_tile) tiles_bbox3857 = self.bbox4326to3857( getattr(top_left_bounds, "west"), getattr(bottom_right_bounds, "east"), getattr(bottom_right_bounds, "south"), getattr(top_left_bounds, "north"), ) # rescale tiles' boundaries - if space covered by the input BBOX extends the width of the world, # (e.g. two "worlds" are present on the map), translation between tiles' BBOX and image's pixel requires # additional rescaling, for tiles' BBOX coordinates to match input BBOX coordinates west_coord = tiles_bbox3857[ 0] + west_rescaling_factor * epsg3857_world_width east_coord = tiles_bbox3857[ 1] + east_rescaling_factor * epsg3857_world_width # prepare translating function from received BBOX to pixel values of the background image src_quad = (0, fixed_top_offset, background.size[0], background.size[1] - fixed_bottom_offset) to_src_px = utils.make_bbox_to_pixels_transf( [west_coord, tiles_bbox3857[2], east_coord, tiles_bbox3857[3]], src_quad) # translate received BBOX to pixel values minx, miny = to_src_px(bbox[0], bbox[2]) maxx, maxy = to_src_px(bbox[1], bbox[3]) # max and min function for Y axis were introduced to mitigate rounding errors crop_box = ( ceil(minx), max(ceil(maxy) + fixed_top_offset, 0), floor(maxx), min(floor(miny) + fixed_top_offset, background.size[1]), ) if not all([ 0 <= crop_x <= background.size[0] for crop_x in [crop_box[0], crop_box[2]] ]): raise ThumbnailError( f"Tiled background cropping error. Boundaries outside of the image: {crop_box}" ) # crop background image to the desired bbox and resize it background = background.crop(box=crop_box) background = background.resize( (self.thumbnail_width, self.thumbnail_height)) if sum(background.convert("L").getextrema()) in (0, 2): # either all black or all white logger.error("Thumbnail background outside the allowed area.") raise ThumbnailError( "Thumbnail background outside the allowed area.") return background
def geoserver_post_save(instance, sender, **kwargs): """Save keywords to GeoServer The way keywords are implemented requires the layer to be saved to the database before accessing them. """ url = ogc_server_settings.internal_rest try: gs_resource = gs_catalog.get_resource(instance.name) except (FailedRequestError, EnvironmentError) as e: msg = ('Could not connect to geoserver at "%s"' 'to save information for layer "%s"' % (ogc_server_settings.LOCATION, instance.name.encode('utf-8'))) logger.warn(msg, e) # If geoserver is not online, there is no need to continue return # If there is no resource returned it could mean one of two things: # a) There is a synchronization problem in geoserver # b) The unit tests are running and another geoserver is running in the # background. # For both cases it is sensible to stop processing the layer if gs_resource is None: logger.warn('Could not get geoserver resource for %s' % instance) return gs_resource.keywords = instance.keyword_list() #gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) bbox = gs_resource.latlon_bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) dataAspect = 1 if dy == 0 else dx / dy height = 550 width = int(height * dataAspect) # Set download links for WMS, WCS or WFS and KML links = wms_links(ogc_server_settings.public_url + 'wms?', instance.typename.encode('utf-8'), instance.bbox_string, instance.srid, height, width) for ext, name, mime, wms_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, name=ugettext(name), defaults=dict( extension=ext, url=wms_url, mime=mime, link_type='image', )) if instance.storeType == "dataStore": links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8')) for ext, name, mime, wfs_url in links: if mime == 'SHAPE-ZIP': name = 'Zipped Shapefile' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wfs_url, defaults=dict( extension=ext, name=name, mime=mime, url=wfs_url, link_type='data', )) elif instance.storeType == 'coverageStore': #FIXME(Ariel): This works for public layers, does it work for restricted too? # would those end up with no geotiff links, like, forever? permissions = {} permissions['anonymous'] = instance.get_gen_level(ANONYMOUS_USERS) permissions['authenticated'] = instance.get_gen_level( AUTHENTICATED_USERS) instance.set_gen_level(ANONYMOUS_USERS, 'layer_readonly') #Potentially 3 dimensions can be returned by the grid if there is a z #axis. Since we only want width/height, slice to the second dimension covWidth, covHeight = get_coverage_grid_extent(instance)[:2] links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8'), bbox=gs_resource.native_bbox[:-1], crs=gs_resource.native_bbox[-1], height=str(covHeight), width=str(covWidth)) for ext, name, mime, wcs_url in links: Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=wcs_url, defaults=dict( extension=ext, name=name, mime=mime, link_type='data', )) instance.set_gen_level(ANONYMOUS_USERS, permissions['anonymous']) instance.set_gen_level(AUTHENTICATED_USERS, permissions['authenticated']) kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode( { 'layers': instance.typename.encode('utf-8'), 'mode': "download" }) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_download, defaults=dict( extension='kml', name=_("KML"), mime='text/xml', link_type='data', )) kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode( { 'layers': instance.typename.encode('utf-8'), 'mode': "refresh" }) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=kml_reflector_link_view, defaults=dict( extension='kml', name="View in Google Earth", mime='text/xml', link_type='data', )) tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url + 'layers=%s' % instance.typename.encode('utf-8') + '&zoom={z}&x={x}&y={y}' + '&format=image/png8') Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=tile_url, defaults=dict( extension='tiles', name=_("Tiles"), mime='image/png', link_type='image', )) wms_path = '%s/%s/wms' % (instance.workspace, instance.name) ows_url = urljoin(ogc_server_settings.public_url, wms_path) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ows_url, defaults=dict( extension='html', name=_("OWS"), url=ows_url, mime='text/html', link_type='OGC:WMS', )) html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=html_link_url, defaults=dict( extension='html', name=instance.typename, mime='text/html', link_type='html', )) params = { 'layers': instance.typename.encode('utf-8'), 'format': 'image/png8', 'width': 200, 'height': 150, } # Avoid using urllib.urlencode here because it breaks the url. # commas and slashes in values get encoded and then cause trouble # with the WMS parser. p = "&".join("%s=%s" % item for item in params.items()) thumbnail_url = ogc_server_settings.LOCATION + "wms/reflect?" + p Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=thumbnail_url, defaults=dict( extension='png', name=_("Remote Thumbnail"), mime='image/png', link_type='image', )) # Download thumbnail and save it locally. resp, image = http_client.request(thumbnail_url) if 'ServiceException' in image or resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.debug(msg) # Replace error message with None. image = None if image is not None: #Clean any orphan Thumbnail before Thumbnail.objects.filter(resourcebase__id=None).delete() thumbnail, created = Thumbnail.objects.get_or_create( resourcebase__id=instance.id) thumbnail.thumb_spec = thumbnail_url thumbnail.save_thumb(image, instance._thumbnail_path()) Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=settings.SITEURL + instance._thumbnail_path(), defaults=dict( extension='png', name=_("Thumbnail"), mime='image/png', link_type='image', )) ogc_wms_url = ogc_server_settings.public_url + 'wms?' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wms_url, defaults=dict( extension='html', name=instance.name, url=ogc_wms_url, mime='text/html', link_type='OGC:WMS', )) if instance.storeType == "dataStore": ogc_wfs_url = ogc_server_settings.public_url + 'wfs?' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wfs_url, defaults=dict( extension='html', name=instance.name, url=ogc_wfs_url, mime='text/html', link_type='OGC:WFS', )) if instance.storeType == "coverageStore": ogc_wcs_url = ogc_server_settings.public_url + 'wcs?' Link.objects.get_or_create(resource=instance.resourcebase_ptr, url=ogc_wcs_url, defaults=dict( extension='html', name=instance.name, url=ogc_wcs_url, mime='text/html', link_type='OGC:WCS', )) #remove links that belong to and old address for link in instance.link_set.all(): if not urlparse(settings.SITEURL).hostname == urlparse(link.url).hostname and not \ urlparse(ogc_server_settings.public_url).hostname == urlparse(link.url).hostname: link.delete() #Save layer attributes set_attributes(instance) #Save layer styles set_styles(instance, gs_catalog)