def layer_batch_download(request):
"""
batch download a set of layers
POST - begin download
GET?id=<download_id> monitor status
"""
from geonode.utils import http_client, _get_basic_auth_info
# currently this just piggy-backs on the map download backend
# by specifying an ad hoc map that contains all layers requested
# for download. assumes all layers are hosted locally.
# status monitoring is handled slightly differently.
if request.method == 'POST':
layers = request.POST.getlist("layer")
layers = Layer.objects.filter(typename__in=list(layers))
def layer_son(layer):
return {
"name": layer.typename,
"service": layer.service_type,
"metadataURL": "",
"serviceURL": ""
}
readme = """This data is provided by GeoNode.\n\nContents:"""
def list_item(lyr):
return "%s - %s.*" % (lyr.title, lyr.name)
readme = "\n".join([readme] + [list_item(l) for l in layers])
fake_map = {
"map": {
"readme": readme
},
"layers": [layer_son(lyr) for lyr in layers]
}
url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
resp, content = http_client.request(url,
'POST',
body=json.dumps(fake_map))
return HttpResponse(content, status=resp.status)
if request.method == 'GET':
# essentially, this just proxies back to geoserver
download_id = request.GET.get('id', None)
if download_id is None:
return HttpResponse(status=404)
url = "%srest/process/batchDownload/status/%s" % (
ogc_server_settings.LOCATION, download_id)
resp, content = http_client.request(url, 'GET')
return HttpResponse(content, status=resp.status)
def layer_batch_download(request):
"""
batch download a set of layers
POST - begin download
GET?id=<download_id> monitor status
"""
from geonode.utils import http_client
# currently this just piggy-backs on the map download backend
# by specifying an ad hoc map that contains all layers requested
# for download. assumes all layers are hosted locally.
# status monitoring is handled slightly differently.
if request.method == 'POST':
layers = request.POST.getlist("layer")
layers = Layer.objects.filter(typename__in=list(layers))
def layer_son(layer):
return {
"name": layer.typename,
"service": layer.service_type,
"metadataURL": "",
"serviceURL": ""
}
readme = """This data is provided by GeoNode.\n\nContents:"""
def list_item(lyr):
return "%s - %s.*" % (lyr.title, lyr.name)
readme = "\n".join([readme] + [list_item(l) for l in layers])
fake_map = {
"map": {"readme": readme},
"layers": [layer_son(lyr) for lyr in layers]
}
url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
resp, content = http_client.request(
url, 'POST', body=json.dumps(fake_map))
return HttpResponse(content, status=resp.status)
if request.method == 'GET':
# essentially, this just proxies back to geoserver
download_id = request.GET.get('id', None)
if download_id is None:
return HttpResponse(status=404)
url = "%srest/process/batchDownload/status/%s" % (
ogc_server_settings.LOCATION, download_id)
resp, content = http_client.request(url, 'GET')
return HttpResponse(content, status=resp.status)
def probe_service(self):
from geonode.utils import http_client
try:
resp, content = http_client.request(self.service_url)
return resp.status_code
except Exception:
return 404
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
mapObject = _resolve_map(request, mapid, 'maps.view_map')
map_status = dict()
if request.method == 'POST':
url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
def perm_filter(layer):
return request.user.has_perm('layers.view_layer', obj=layer)
mapJson = mapObject.json(perm_filter)
# we need to remove duplicate layers
j_map = json.loads(mapJson)
j_layers = j_map["layers"]
for j_layer in j_layers:
if(len([l for l in j_layers if l == j_layer]))>1:
j_layers.remove(j_layer)
mapJson = json.dumps(j_map)
resp, content = http_client.request(url, 'POST', body=mapJson)
status = int(resp.status)
if status == 200:
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
raise Exception('Could not start the download of %s. Error was: %s' % (mapObject.title, content))
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in mapObject.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(typename=lyr.name)
if not request.user.has_perm('layers.view_layer', obj=ownable_layer):
locked_layers.append(lyr)
else:
# we need to add the layer only once
if len([l for l in downloadable_layers if l.name == lyr.name]) == 0:
downloadable_layers.append(lyr)
return render_to_response(template, RequestContext(request, {
"map_status" : map_status,
"map" : mapObject,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"site" : settings.SITEURL
}))
def _fetch_thumb_and_compare(self, url, expected_image):
if url == missing_thumbnail_url:
logger.error(f'It was not possible to fetch the remote layer WMS GetMap! thumb_url: {url}')
return
_, img = http_client.request(url)
content = BytesIO(img)
Image.open(content).verify() # verify that it is, in fact an image
thumb = Image.open(content)
diff = Image.new("RGB", thumb.size)
mismatch = pixelmatch(thumb, expected_image, diff)
if mismatch >= expected_image.size[0] * expected_image.size[1] * 0.01:
logger.warn("Mismatch, it was not possible to bump the bg!")
# Sometimes this test fails to fetch the OSM background
with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.png', delete=False) as tmpfile:
logger.error(f"Dumping thumb to: {tmpfile.name}")
thumb.save(tmpfile)
# Let's check that the thumb is valid at least
with Image.open(tmpfile) as img:
img.verify()
with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.png', delete=False) as tmpfile:
logger.error(f"Dumping diff to: {tmpfile.name}")
diff.save(tmpfile)
# Let's check that the thumb is valid at least
with Image.open(tmpfile) as img:
img.verify()
else:
self.assertTrue(
mismatch < expected_image.size[0] * expected_image.size[1] * 0.01,
"Expected test and pre-generated thumbnails to differ up to 1%",
)
def probe(self):
from geonode.utils import http_client
try:
resp, _ = http_client.request(self.url)
return resp.status_code in (200, 201)
except Exception:
return False
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
mapObject = _resolve_map(request, mapid, 'maps.view_map')
map_status = dict()
if request.method == 'POST':
url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
def perm_filter(layer):
return request.user.has_perm('layers.view_layer', obj=layer)
mapJson = mapObject.json(perm_filter)
# we need to remove duplicate layers
j_map = json.loads(mapJson)
j_layers = j_map["layers"]
for j_layer in j_layers:
if(len([l for l in j_layers if l == j_layer]))>1:
j_layers.remove(j_layer)
mapJson = json.dumps(j_map)
resp, content = http_client.request(url, 'POST', body=mapJson)
if resp.status not in (400, 404, 417):
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
pass # XXX fix
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in mapObject.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(typename=lyr.name)
if not request.user.has_perm('layers.view_layer', obj=ownable_layer):
locked_layers.append(lyr)
else:
# we need to add the layer only once
if len([l for l in downloadable_layers if l.name == lyr.name]) == 0:
downloadable_layers.append(lyr)
return render_to_response(template, RequestContext(request, {
"map_status" : map_status,
"map" : mapObject,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"geoserver" : ogc_server_settings.public_url,
"site" : settings.SITEURL
}))
def _fetch_wp_json():
base_url = settings.BLOG_BASE_URL
wp_json_url = base_url + "/wp-json/wp/v2/posts?_embed&page=1&per_page=3"
response, content = http_client.request(wp_json_url, method='GET')
_output_json = json.loads(content)
for _item in _output_json:
_item['image_url'] = _item['_embedded']['wp:featuredmedia'][0][
'source_url']
return {'wp_posts': _output_json}
def create_thumbnail(instance, thumbnail_remote_url, thumbail_create_url=None):
BBOX_DIFFERENCE_THRESHOLD = 1e-5
if not thumbail_create_url:
thumbail_create_url = thumbnail_remote_url
# Check if the bbox is invalid
valid_x = (float(instance.bbox_x0) -
float(instance.bbox_x1))**2 > BBOX_DIFFERENCE_THRESHOLD
valid_y = (float(instance.bbox_y1) -
float(instance.bbox_y0))**2 > BBOX_DIFFERENCE_THRESHOLD
image = None
if valid_x and valid_y:
Link.objects.get_or_create(resource=instance.get_self_resource(),
url=thumbnail_remote_url,
defaults=dict(
extension='png',
name=_("Remote Thumbnail"),
mime='image/png',
link_type='image',
))
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbail_create_url)
if 'ServiceException' in image or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
# first delete thumbnail file on disk to prevent duplicates:
if instance.has_thumbnail():
instance.thumbnail_set.get().thumb_file.delete()
# update database and save new thumbnail file on disk:
instance.thumbnail_set.all().delete()
thumbnail = Thumbnail(thumb_spec=thumbnail_remote_url)
instance.thumbnail_set.add(thumbnail)
thumbnail.thumb_file.save('layer-%s-thumb.png' % instance.id,
ContentFile(image))
thumbnail.save()
thumbnail_url = urljoin(settings.SITEURL,
instance.thumbnail_set.get().thumb_file.url)
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(
name=_('Thumbnail'),
extension='png',
mime='image/png',
link_type='image',
))
ResourceBase.objects.filter(id=instance.id).update(
thumbnail_url=instance.get_thumbnail_url())
def geoserver_post_save_map(instance, sender, **kwargs):
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(
Layer.objects.get(typename=layer.name).typename)
params = {
'layers': ",".join(local_layers).encode('utf-8'),
'format': 'image/png8',
'width': 200,
'height': 150,
}
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_url = ogc_server_settings.LOCATION + "wms/reflect?" + p
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(
extension='png',
name=_("Remote Thumbnail"),
mime='image/png',
link_type='image',
))
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbnail_url)
if 'ServiceException' in image or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
#Clean any orphan Thumbnail before
Thumbnail.objects.filter(resourcebase__id=None).delete()
thumbnail, created = Thumbnail.objects.get_or_create(
resourcebase__id=instance.id)
thumbnail.thumb_spec = thumbnail_url
thumbnail.save_thumb(image, instance._thumbnail_path())
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=settings.SITEURL +
instance._thumbnail_path(),
defaults=dict(
extension='png',
name=_("Thumbnail"),
mime='image/png',
link_type='image',
))
def geoserver_post_save_map(instance, sender, **kwargs):
instance.set_missing_info()
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(Layer.objects.get(typename=layer.name).typename)
image = None
# If the map does not have any local layers, do not create the thumbnail.
if len(local_layers) > 0:
params = {"layers": ",".join(local_layers).encode("utf-8"), "format": "image/png8", "width": 200, "height": 150}
# Add the bbox param only if the bbox is different to [None, None, None, None]
if None not in instance.bbox:
params["bbox"] = instance.bbox_string
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_remote_url = ogc_server_settings.LOCATION + "wms/reflect?" + p
Link.objects.get_or_create(
resource=instance.resourcebase_ptr,
url=thumbnail_remote_url,
defaults=dict(extension="png", name=_("Remote Thumbnail"), mime="image/png", link_type="image"),
)
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbnail_remote_url)
if "ServiceException" in image or resp.status < 200 or resp.status > 299:
msg = "Unable to obtain thumbnail: %s" % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
if instance.has_thumbnail():
instance.thumbnail.thumb_file.delete()
else:
instance.thumbnail = Thumbnail()
instance.thumbnail.thumb_file.save("map-%s-thumb.png" % instance.id, ContentFile(image))
instance.thumbnail.thumb_spec = thumbnail_remote_url
instance.thumbnail.save()
thumbnail_url = urljoin(settings.SITEURL, instance.thumbnail.thumb_file.url)
Link.objects.get_or_create(
resource=instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(name=_("Thumbnail"), extension="png", mime="image/png", link_type="image"),
)
def fetch_wms(url: str, max_retries: int = 3, retry_delay: int = 1):
"""
Function fetching an image from OGC server. The request is performed based on the WMS URL.
In case access_token in not present in the URL , and Geoserver is used and the OGC backend, Basic Authentication
is used instead. If image retrieval fails, function retries to fetch the image max_retries times, waiting
retry_delay seconds between consecutive requests.
:param url: WMS URL of the image
:param max_retries: maximum number of retries before skipping retrieval
:param retry_delay: number of seconds waited between retries
:returns: retrieved image
"""
# prepare authorization for WMS service
headers = {}
if "access_token" not in url:
if url.startswith(settings.OGC_SERVER["default"]["LOCATION"]):
# for the Geoserver backend, use Basic Auth, if access_token is not provided
_user = settings.OGC_SERVER["default"].get("USER")
_pwd = settings.OGC_SERVER["default"].get("PASSWORD")
encoded_credentials = base64.b64encode(
f"{_user}:{_pwd}".encode("UTF-8")).decode("ascii")
headers["Authorization"] = f"Basic {encoded_credentials}"
image = None
for retry in range(max_retries):
try:
# fetch data
resp, image = http_client.request(
url,
headers=headers,
timeout=settings.OGC_SERVER["default"].get("TIMEOUT", 60))
# validate response
if not resp or resp.status_code < 200 or resp.status_code > 299 or "ServiceException" in str(
image):
_status_code = resp.status_code if resp else 'Unknown'
logger.debug(
f"Fetching partial thumbnail from {url} failed with status code: "
f"{_status_code} and response: {str(image)}")
image = None
time.sleep(retry_delay)
continue
except Exception as e:
if retry + 1 >= max_retries:
logger.exception(e)
time.sleep(retry_delay)
continue
else:
break
return image
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
mapObject = _resolve_map(request, mapid, 'maps.view_map')
map_status = dict()
if request.method == 'POST':
url = "%srest/process/batchDownload/launch/" % settings.GEOSERVER_BASE_URL
def perm_filter(layer):
return request.user.has_perm('maps.view_layer', obj=layer)
mapJson = mapObject.json(perm_filter)
resp, content = http_client.request(url, 'POST', body=mapJson)
if resp.status not in (400, 404, 417):
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
pass # XXX fix
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in mapObject.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(typename=lyr.name)
if not request.user.has_perm('maps.view_layer',
obj=ownable_layer):
locked_layers.append(lyr)
else:
downloadable_layers.append(lyr)
return render_to_response(
template,
RequestContext(
request, {
"map_status": map_status,
"map": mapObject,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"geoserver": settings.GEOSERVER_BASE_URL,
"site": settings.SITEURL
}))
def get_gs_thumbnail(instance):
from geonode.geoserver.helpers import ogc_server_settings
if instance.class_name == 'Map':
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(layer.name)
layers = ",".join(local_layers).encode('utf-8')
if (len(local_layers) == 0):
return None
else:
layers = instance.typename.encode('utf-8')
params = {
'layers': layers,
'format': 'image/png8',
'width': 200,
'height': 150,
'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z'
}
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_create_url = ogc_server_settings.LOCATION + \
"wms/reflect?" + p
tries = 0
max_tries = 30
while tries < max_tries:
logger.debug(
'Thumbnail: Requesting thumbnail from GeoServer. '
'Attempt %d of %d', tries + 1, max_tries)
resp, image = http_client.request(thumbnail_create_url)
if 200 <= resp.status <= 299:
if 'ServiceException' not in image:
return image
else:
# Unexpected Error Code, Stop Trying
logger.debug(
'Thumbnail: Encountered unexpected status code: %d. '
'Aborting.', resp.status)
logger.debug(resp)
break
# Layer not ready yet, try again
tries += 1
time.sleep(1)
return None
def create_thumbnail(instance, thumbnail_remote_url):
BBOX_DIFFERENCE_THRESHOLD = 1e-5
#Check if the bbox is invalid
valid_x = (float(instance.bbox_x0) - float(instance.bbox_x1))**2 > BBOX_DIFFERENCE_THRESHOLD
valid_y = (float(instance.bbox_y1) - float(instance.bbox_y0))**2 > BBOX_DIFFERENCE_THRESHOLD
image = None
if valid_x and valid_y:
Link.objects.get_or_create(resource= instance.get_self_resource(),
url=thumbnail_remote_url,
defaults=dict(
extension='png',
name=_("Remote Thumbnail"),
mime='image/png',
link_type='image',
)
)
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbnail_remote_url)
if 'ServiceException' in image or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
if instance.has_thumbnail():
instance.thumbnail.thumb_file.delete()
else:
instance.thumbnail = Thumbnail()
instance.thumbnail.thumb_file.save('layer-%s-thumb.png' % instance.id, ContentFile(image))
instance.thumbnail.thumb_spec = thumbnail_remote_url
instance.thumbnail.save()
thumbnail_url = urljoin(settings.SITEURL, instance.thumbnail.thumb_file.url)
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(
name=_('Thumbnail'),
extension='png',
mime='image/png',
link_type='image',
)
)
ResourceBase.objects.filter(id=instance.id).update(
thumbnail_url=instance.get_thumbnail_url()
)
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
mapObject = _resolve_map(request, mapid, 'maps.view_map')
map_status = dict()
if request.method == 'POST':
url = "%srest/process/batchDownload/launch/" % settings.GEOSERVER_BASE_URL
def perm_filter(layer):
return request.user.has_perm('maps.view_layer', obj=layer)
mapJson = mapObject.json(perm_filter)
resp, content = http_client.request(url, 'POST', body=mapJson)
if resp.status not in (400, 404, 417):
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
pass # XXX fix
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in mapObject.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(typename=lyr.name)
if not request.user.has_perm('maps.view_layer', obj=ownable_layer):
locked_layers.append(lyr)
else:
downloadable_layers.append(lyr)
return render_to_response(template, RequestContext(request, {
"map_status" : map_status,
"map" : mapObject,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"geoserver" : settings.GEOSERVER_BASE_URL,
"site" : settings.SITEURL
}))
def test_map_default_thumb(self):
create_gs_thumbnail_geonode(self.map_composition, overwrite=True)
if not self.map_composition.has_thumbnail():
logger.warn("It was not possible to dump the background!")
logger.error(
f"map_composition thumb: {self.map_composition.thumbnail_url}")
else:
_, img = http_client.request(self.map_composition.thumbnail_url)
content = BytesIO(img)
Image.open(content).verify() # verify that it is, in fact an image
thumb = Image.open(content)
diff = Image.new("RGB", thumb.size)
expected_thumb = Image.open(EXPECTED_RESULTS_DIR +
"thumbnails/default_map_thumb.png")
mismatch = pixelmatch(thumb, expected_thumb, diff)
if mismatch >= expected_thumb.size[0] * expected_thumb.size[
1] * 0.01:
logger.warn("Mismatch, it was not possible to bump the bg!")
# Sometimes this test fails to fetch the OSM background
with tempfile.NamedTemporaryFile(dir='/tmp',
suffix='.png',
delete=False) as tmpfile:
logger.error(f"Dumping thumb to: {tmpfile.name}")
thumb.save(tmpfile)
# Let's check that the thumb is valid at least
with Image.open(tmpfile) as img:
img.verify()
with tempfile.NamedTemporaryFile(dir='/tmp',
suffix='.png',
delete=False) as tmpfile:
logger.error(f"Dumping diff to: {tmpfile.name}")
diff.save(tmpfile)
# Let's check that the thumb is valid at least
with Image.open(tmpfile) as img:
img.verify()
else:
self.assertTrue(
mismatch <
expected_thumb.size[0] * expected_thumb.size[1] * 0.01,
"Expected test and pre-generated thumbnails to differ up to 1%",
)
def map_download_check(request):
"""
this is an endpoint for monitoring map downloads
"""
try:
layer = request.session["map_status"]
if type(layer) == dict:
url = "%srest/process/batchDownload/status/%s" % (ogc_server_settings.LOCATION,layer["id"])
resp,content = http_client.request(url,'GET')
status= resp.status
if resp.status == 400:
return HttpResponse(content="Something went wrong",status=status)
else:
content = "Something Went wrong"
status = 400
except ValueError:
# TODO: Is there any useful context we could include in this log?
logger.warn("User tried to check status, but has no download in progress.")
return HttpResponse(content=content,status=status)
def map_download_check(request):
"""
this is an endpoint for monitoring map downloads
"""
try:
layer = request.session["map_status"]
if type(layer) == dict:
url = "%srest/process/batchDownload/status/%s" % (ogc_server_settings.LOCATION,layer["id"])
resp,content = http_client.request(url,'GET')
status= resp.status
if resp.status == 400:
return HttpResponse(content="Something went wrong",status=status)
else:
content = "Something Went wrong"
status = 400
except ValueError:
# TODO: Is there any useful context we could include in this log?
logger.warn("User tried to check status, but has no download in progress.")
return HttpResponse(content=content,status=status)
def get_gs_thumbnail(instance):
from geonode.geoserver.helpers import ogc_server_settings
if instance.class_name == 'Map':
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(layer.name)
layers = ",".join(local_layers).encode('utf-8')
if (len(local_layers) == 0):
return None
else:
layers = instance.typename.encode('utf-8')
params = {
'layers': layers,
'format': 'image/png8',
'width': 200,
'height': 150,
'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z'
}
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_create_url = ogc_server_settings.LOCATION + \
"wms/reflect?" + p
resp, image = http_client.request(thumbnail_create_url)
if ('ServiceException' in image or resp.status < 200 or resp.status > 299):
return None
return image
def _enrich_layer_metadata(self, geonode_service, geonode_layer):
workspace, layername = geonode_layer.name.split(
":") if ":" in geonode_layer.name else (None, geonode_layer.name)
url = urlsplit(self.url)
base_url = f'{url.scheme}://{url.netloc}/'
response = requests.get(f'{base_url}api/layers/?name={layername}', {},
timeout=10,
verify=False)
content = response.content
status = response.status_code
content_type = response.headers['Content-Type']
if status == 200 and 'application/json' == content_type:
try:
if isinstance(content, bytes):
content = content.decode('UTF-8')
_json_obj = json.loads(content)
if _json_obj['meta']['total_count'] == 1:
_layer = _json_obj['objects'][0]
if _layer:
r_fields = {}
# Update plain fields
for field in GeoNodeServiceHandler.LAYER_FIELDS:
if field in _layer and _layer[field]:
r_fields[field] = _layer[field]
if r_fields:
Layer.objects.filter(id=geonode_layer.id).update(
**r_fields)
geonode_layer.refresh_from_db()
# Update Thumbnail
if "thumbnail_url" in _layer and _layer[
"thumbnail_url"]:
thumbnail_remote_url = _layer["thumbnail_url"]
_url = urlsplit(thumbnail_remote_url)
if not _url.scheme:
thumbnail_remote_url = f"{geonode_layer.remote_service.service_url}{_url.path}"
resp, image = http_client.request(
thumbnail_remote_url)
if 'ServiceException' in str(image) or \
resp.status_code < 200 or resp.status_code > 299:
msg = f'Unable to obtain thumbnail: {image}'
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
thumbnail_name = f'layer-{geonode_layer.uuid}-thumb.png'
geonode_layer.save_thumbnail(thumbnail_name,
image=image)
else:
self._create_layer_thumbnail(geonode_layer)
else:
self._create_layer_thumbnail(geonode_layer)
# Add Keywords
if "keywords" in _layer and _layer["keywords"]:
keywords = _layer["keywords"]
if keywords:
geonode_layer.keywords.clear()
geonode_layer.keywords.add(*keywords)
# Add Regions
if "regions" in _layer and _layer["regions"]:
(regions_resolved,
regions_unresolved) = resolve_regions(
_layer["regions"])
if regions_resolved:
geonode_layer.regions.clear()
geonode_layer.regions.add(*regions_resolved)
# Remove temporary topic category here as the field converted into m2m field
# any idea how to setup?
# Add Topic Category
# if "category" in _layer and _layer["category"]:
# (categories_resolved, categories_unresolved) = resolve_categories(_layer["category"])
# if categories_resolved:
# geonode_layer.category.clear()
# geonode_layer.category.add(*categories_resolved)
# if "category__gn_description" in _layer and _layer["category__gn_description"]:
# try:
# categories = TopicCategory.objects.filter(
# Q(gn_description__iexact=_layer["category__gn_description"]))
# if categories:
# geonode_layer.category = categories[0]
# except Exception:
# traceback.print_exc()
except Exception:
traceback.print_exc()
finally:
try:
geonode_layer.save(notify=True)
except Exception as e:
logger.error(e)
def proxy(request,
url=None,
response_callback=None,
sec_chk_hosts=True,
sec_chk_rules=True,
timeout=None,
allowed_hosts=[],
**kwargs):
# Request default timeout
if not timeout:
timeout = TIMEOUT
# Security rules and settings
PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())
# Sanity url checks
if 'url' not in request.GET and not url:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain")
raw_url = url or request.GET['url']
raw_url = urljoin(settings.SITEURL,
raw_url) if raw_url.startswith("/") else raw_url
url = urlsplit(raw_url)
scheme = str(url.scheme)
locator = str(url.path)
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
# White-Black Listing Hosts
site_url = urlsplit(settings.SITEURL)
if sec_chk_hosts and not settings.DEBUG:
# Attach current SITEURL
if site_url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (site_url.hostname, )
# Attach current hostname
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
from geonode.geoserver.helpers import ogc_server_settings
hostname = (
ogc_server_settings.hostname, ) if ogc_server_settings else ()
if hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += hostname
# Check OWS regexp
if url.query and ows_regexp.match(url.query):
ows_tokens = ows_regexp.match(url.query).groups()
if len(
ows_tokens
) == 4 and 'version' == ows_tokens[0] and StrictVersion(
ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion(
ows_tokens[1]
) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in (
'getcapabilities') and ows_tokens[3].upper() in (
'OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'):
if url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (url.hostname, )
# Check Remote Services base_urls
from geonode.services.models import Service
for _s in Service.objects.all():
_remote_host = urlsplit(_s.base_url).hostname
PROXY_ALLOWED_HOSTS += (_remote_host, )
if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
return HttpResponse(
"DEBUG is set to False but the host of the path provided to the proxy service"
" is not in the PROXY_ALLOWED_HOSTS setting.",
status=403,
content_type="text/plain")
# Security checks based on rules; allow only specific requests
if sec_chk_rules:
# TODO: Not yet implemented
pass
# Collecting headers and cookies
headers, access_token = get_headers(request,
url,
raw_url,
allowed_hosts=allowed_hosts)
# Inject access_token if necessary
parsed = urlparse(raw_url)
parsed._replace(path=locator.encode('utf8'))
if parsed.netloc == site_url.netloc and scheme != site_url.scheme:
parsed = parsed._replace(scheme=site_url.scheme)
_url = parsed.geturl()
# Some clients / JS libraries generate URLs with relative URL paths, e.g.
# "http://host/path/path/../file.css", which the requests library cannot
# currently handle (https://github.com/kennethreitz/requests/issues/2982).
# We parse and normalise such URLs into absolute paths before attempting
# to proxy the request.
_url = URL.from_text(_url).normalize().to_text()
if request.method == "GET" and access_token and 'access_token' not in _url:
query_separator = '&' if '?' in _url else '?'
_url = ('%s%saccess_token=%s' % (_url, query_separator, access_token))
_data = request.body.decode('utf-8')
# Avoid translating local geoserver calls into external ones
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
from geonode.geoserver.helpers import ogc_server_settings
_url = _url.replace('%s%s' % (settings.SITEURL, 'geoserver'),
ogc_server_settings.LOCATION.rstrip('/'))
_data = _data.replace('%s%s' % (settings.SITEURL, 'geoserver'),
ogc_server_settings.LOCATION.rstrip('/'))
response, content = http_client.request(_url,
method=request.method,
data=_data,
headers=headers,
timeout=timeout,
user=request.user)
content = response.content or response.reason
status = response.status_code
content_type = response.headers.get('Content-Type')
if status >= 400:
return HttpResponse(content=content,
reason=content,
status=status,
content_type=content_type)
# decompress GZipped responses if not enabled
# if content and response and response.getheader('Content-Encoding') == 'gzip':
if content and content_type and content_type == 'gzip':
buf = io.BytesIO(content)
f = gzip.GzipFile(fileobj=buf)
content = f.read()
PLAIN_CONTENT_TYPES = ['text', 'plain', 'html', 'json', 'xml', 'gml']
for _ct in PLAIN_CONTENT_TYPES:
if content_type and _ct in content_type and not isinstance(
content, six.string_types):
try:
content = content.decode()
break
except Exception:
pass
if response and response_callback:
kwargs = {} if not kwargs else kwargs
kwargs.update({
'response': response,
'content': content,
'status': status,
'content_type': content_type
})
return response_callback(**kwargs)
else:
# If we get a redirect, let's add a useful message.
if status and status in (301, 302, 303, 307):
_response = HttpResponse(
('This proxy does not support redirects. The server in "%s" '
'asked for a redirect to "%s"' %
(url, response.getheader('Location'))),
status=status,
content_type=content_type)
_response['Location'] = response.getheader('Location')
return _response
else:
def _get_message(text):
_s = text
if isinstance(text, bytes):
_s = text.decode("utf-8", "replace")
try:
found = re.search('<b>Message</b>(.+?)</p>',
_s).group(1).strip()
except Exception:
found = _s
return found
return HttpResponse(
content=content,
reason=_get_message(content) if status not in (200,
201) else None,
status=status,
content_type=content_type)
def _render_thumbnail(self, spec):
resp, content = http_client.request(spec)
if resp.status < 200 or resp.status > 299:
logger.warning('Unable to obtain thumbnail: %s', content)
return
return content
def geoserver_post_save_map(instance, sender, **kwargs):
instance.set_missing_info()
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(layer.name)
image = None
# If the map does not have any local layers, do not create the thumbnail.
if len(local_layers) > 0:
params = {
'layers': ",".join(local_layers).encode('utf-8'),
'format': 'image/png8',
'width': 200,
'height': 150,
}
# Add the bbox param only if the bbox is different to [None, None,
# None, None]
if None not in instance.bbox:
params['bbox'] = instance.bbox_string
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_remote_url = ogc_server_settings.LOCATION + \
"wms/reflect?" + p
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_remote_url,
defaults=dict(
extension='png',
name=_("Remote Thumbnail"),
mime='image/png',
link_type='image',
)
)
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbnail_remote_url)
if 'ServiceException' in image or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
if instance.has_thumbnail():
instance.thumbnail_set.get().thumb_file.delete()
else:
instance.thumbnail_set.add(Thumbnail())
instance.thumbnail_set.get().thumb_file.save(
'map-%s-thumb.png' %
instance.id,
ContentFile(image))
instance.thumbnail_set.get().thumb_spec = thumbnail_remote_url
instance.thumbnail_set.get().save()
thumbnail_url = urljoin(
settings.SITEURL,
instance.thumbnail_set.get().thumb_file.url)
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(
name=_('Thumbnail'),
extension='png',
mime='image/png',
link_type='image',
)
)
from geonode.maps.models import Map
Map.objects.filter(id=instance.id).update(thumbnail_url=thumbnail_url)
def proxy(request, url=None, response_callback=None,
sec_chk_hosts=True, sec_chk_rules=True, timeout=None, **kwargs):
# Request default timeout
if not timeout:
timeout = TIMEOUT
# Security rules and settings
PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())
# Sanity url checks
if 'url' not in request.GET and not url:
return HttpResponse("The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain"
)
raw_url = url or request.GET['url']
raw_url = urljoin(
settings.SITEURL,
raw_url) if raw_url.startswith("/") else raw_url
url = urlsplit(raw_url)
locator = str(url.path)
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
# White-Black Listing Hosts
if sec_chk_hosts and not settings.DEBUG:
site_url = urlsplit(settings.SITEURL)
if site_url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (site_url.hostname, )
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
from geonode.geoserver.helpers import ogc_server_settings
hostname = (
ogc_server_settings.hostname,
) if ogc_server_settings else ()
if hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += hostname
if url.query and ows_regexp.match(url.query):
ows_tokens = ows_regexp.match(url.query).groups()
if len(ows_tokens) == 4 and 'version' == ows_tokens[0] and StrictVersion(
ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion(
ows_tokens[1]) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in (
'getcapabilities') and ows_tokens[3].upper() in ('OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'):
if url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (url.hostname, )
if not validate_host(
url.hostname, PROXY_ALLOWED_HOSTS):
return HttpResponse("DEBUG is set to False but the host of the path provided to the proxy service"
" is not in the PROXY_ALLOWED_HOSTS setting.",
status=403,
content_type="text/plain"
)
# Security checks based on rules; allow only specific requests
if sec_chk_rules:
# TODO: Not yet implemented
pass
# Collecting headers and cookies
headers, access_token = get_headers(request, url, raw_url)
# Inject access_token if necessary
parsed = urlparse(raw_url)
parsed._replace(path=locator.encode('utf8'))
_url = parsed.geturl()
if request.method == "GET" and access_token and 'access_token' not in _url:
query_separator = '&' if '?' in _url else '?'
_url = ('%s%saccess_token=%s' %
(_url, query_separator, access_token))
response, content = http_client.request(_url,
method=request.method,
data=request.body,
headers=headers,
timeout=timeout,
user=request.user)
content = response.content or response.reason
status = response.status_code
content_type = response.headers.get('Content-Type')
# decompress GZipped responses if not enabled
# if content and response and response.getheader('Content-Encoding') == 'gzip':
if content and content_type and content_type == 'gzip':
from StringIO import StringIO
import gzip
buf = StringIO(content)
f = gzip.GzipFile(fileobj=buf)
content = f.read()
if response and response_callback:
kwargs = {} if not kwargs else kwargs
kwargs.update({
'response': response,
'content': content,
'status': status,
'content_type': content_type
})
return response_callback(**kwargs)
else:
# If we get a redirect, let's add a useful message.
if status and status in (301, 302, 303, 307):
_response = HttpResponse(('This proxy does not support redirects. The server in "%s" '
'asked for a redirect to "%s"' % (url, response.getheader('Location'))),
status=status,
content_type=content_type
)
_response['Location'] = response.getheader('Location')
return _response
else:
def _get_message(text):
_s = text.decode("utf-8", "replace")
try:
found = re.search('<b>Message</b>(.+?)</p>', _s).group(1).strip()
except BaseException:
found = _s
return found
return HttpResponse(
content=content,
reason=_get_message(content) if status not in (200, 201) else None,
status=status,
content_type=content_type)
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
if not mapid.isdigit():
map_obj = _resolve_map_custom(request, mapid, 'urlsuffix',
'base.view_resourcebase',
_PERMISSION_MSG_VIEW)
else:
map_obj = _resolve_map(request, mapid, 'base.view_resourcebase',
_PERMISSION_MSG_VIEW)
map_status = dict()
if request.method == 'POST':
url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
def perm_filter(layer):
return request.user.has_perm('base.view_resourcebase',
obj=layer.get_self_resource())
mapJson = map_obj.json(perm_filter)
# we need to remove duplicate layers
j_map = json.loads(mapJson)
j_layers = j_map["layers"]
for j_layer in j_layers:
if (len([l for l in j_layers if l == j_layer])) > 1:
j_layers.remove(j_layer)
mapJson = json.dumps(j_map)
resp, content = http_client.request(url, 'POST', body=mapJson)
status = int(resp.status)
if status == 200:
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
raise Exception(
'Could not start the download of %s. Error was: %s' %
(map_obj.title, content))
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in map_obj.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(typename=lyr.name)
if not request.user.has_perm(
'view_resourcebase',
obj=ownable_layer.get_self_resource()):
locked_layers.append(lyr)
else:
# we need to add the layer only once
if len(
[l for l in downloadable_layers
if l.name == lyr.name]) == 0:
downloadable_layers.append(lyr)
return render_to_response(
template,
RequestContext(
request, {
"map_status": map_status,
"map": map_obj,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"site": settings.SITEURL
}))
def _enrich_layer_metadata(self, geonode_layer):
url = urlsplit(self.url)
if url.scheme == 'https':
conn = HTTPSConnection(url.hostname, url.port)
else:
conn = HTTPConnection(url.hostname, url.port)
workspace, layername = geonode_layer.name.split(
":") if ":" in geonode_layer.name else (None, geonode_layer.name)
conn.request('GET', '/api/layers/?name=%s' % layername, '', {})
response = conn.getresponse()
content = response.read()
status = response.status
content_type = response.getheader("Content-Type", "text/plain")
if status == 200 and 'application/json' == content_type:
try:
_json_obj = json.loads(content)
if _json_obj['meta']['total_count'] == 1:
_layer = _json_obj['objects'][0]
if _layer:
r_fields = {}
# Update plain fields
for field in GeoNodeServiceHandler.LAYER_FIELDS:
if field in _layer and _layer[field]:
r_fields[field] = _layer[field]
if r_fields:
Layer.objects.filter(
id=geonode_layer.id).update(
**r_fields)
geonode_layer.refresh_from_db()
# Update Thumbnail
if "thumbnail_url" in _layer and _layer["thumbnail_url"]:
thumbnail_remote_url = _layer["thumbnail_url"]
_url = urlsplit(thumbnail_remote_url)
if not _url.scheme:
thumbnail_remote_url = "{}{}".format(
geonode_layer.remote_service.service_url, _url.path)
resp, image = http_client.request(
thumbnail_remote_url)
if 'ServiceException' in image or \
resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid
geonode_layer.save_thumbnail(
thumbnail_name, image=image)
else:
self._create_layer_thumbnail(geonode_layer)
# Add Keywords
if "keywords" in _layer and _layer["keywords"]:
keywords = _layer["keywords"]
if keywords:
geonode_layer.keywords.clear()
geonode_layer.keywords.add(*keywords)
# Add Regions
if "regions" in _layer and _layer["regions"]:
(regions_resolved, regions_unresolved) = resolve_regions(
_layer["regions"])
if regions_resolved:
geonode_layer.regions.clear()
geonode_layer.regions.add(*regions_resolved)
# Add Topic Category
if "category__gn_description" in _layer and _layer["category__gn_description"]:
try:
categories = TopicCategory.objects.filter(
Q(gn_description__iexact=_layer["category__gn_description"]))
if categories:
geonode_layer.category = categories[0]
except BaseException:
traceback.print_exc()
except BaseException:
traceback.print_exc()
finally:
geonode_layer.save()
def training_download(request,
id,
template='trainings/training_download.html'):
"""
Download all the layers of a training as a batch
"""
training = get_object_or_404(Training, pk=id)
training_status = dict()
if request.method == 'POST':
url = ("%srest/process/batchDownload/launch/" %
ogc_server_settings.LOCATION)
def perm_filter(layer):
return request.user.has_perm('layers.view_layer', obj=layer)
# here we build the json necessary to the rest batchDownload
data = {
'layers': [],
"map": {
'readme': training.abstract,
'title': training.title
}
}
for layer in training.layers.all():
store_type = 'WFS'
if layer.storeType == 'coverageStore':
store_type = 'WCS'
layer_data = {
'metadataURL': '',
'service': store_type,
'name': layer.typename,
'serviceURL': ''
}
data['layers'].append(layer_data)
training_json = json.dumps(data)
resp, content = http_client.request(url, 'POST', body=training_json)
status = int(resp.status)
if status == 200:
training_status = json.loads(content)
request.session["training_status"] = training_status
else:
raise Exception(
'Could not start the download of %s. Error was: %s' %
(training.title, content))
downloadable_layers = []
for layer in training.layers.all():
if request.user.has_perm('layers.view_layer', obj=layer):
downloadable_layers.append(layer)
return render_to_response(
template,
RequestContext(
request, {
"training_status": training_status,
"training": training,
"downloadable_layers": downloadable_layers,
"geoserver": ogc_server_settings.public_url,
"site": settings.SITEURL
}))
def map_download(request, mapid, template="maps/map_download.html"):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
try:
map_obj = _resolve_map(request, mapid, "base.download_resourcebase",
_PERMISSION_MSG_VIEW)
except PermissionDenied:
return HttpResponse(MSG_NOT_ALLOWED, status=403)
except Exception:
raise Http404(MSG_NOT_FOUND)
if not map_obj:
raise Http404(MSG_NOT_FOUND)
map_status = dict()
if request.method == "POST":
def perm_filter(layer):
return request.user.has_perm("base.view_resourcebase",
obj=layer.get_self_resource())
mapJson = map_obj.json(perm_filter)
# we need to remove duplicate layers
j_map = json.loads(mapJson)
j_datasets = j_map["layers"]
for j_dataset in j_datasets:
if j_dataset["service"] is None:
j_datasets.remove(j_dataset)
continue
if (len([_l for _l in j_datasets if _l == j_dataset])) > 1:
j_datasets.remove(j_dataset)
mapJson = json.dumps(j_map)
# the path to geoserver backend continue here
url = urljoin(settings.SITEURL,
reverse("download-map", kwargs={"mapid": mapid}))
resp, content = http_client.request(url, "POST", data=mapJson)
status = int(resp.status_code)
if status == 200:
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
raise Exception(
f"Could not start the download of {map_obj.title}. Error was: {content}"
)
locked_datasets = []
remote_datasets = []
downloadable_datasets = []
for lyr in map_obj.maplayers.iterator():
if lyr.group != "background":
if not lyr.local:
remote_datasets.append(lyr)
else:
ownable_dataset = Dataset.objects.get(alternate=lyr.name)
if not request.user.has_perm(
"download_resourcebase",
obj=ownable_dataset.get_self_resource()):
locked_datasets.append(lyr)
else:
# we need to add the layer only once
if len([
_l for _l in downloadable_datasets
if _l.name == lyr.name
]) == 0:
downloadable_datasets.append(lyr)
site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith(
"http") else settings.SITEURL
register_event(request, EventType.EVENT_DOWNLOAD, map_obj)
return render(
request,
template,
context={
"geoserver": ogc_server_settings.PUBLIC_LOCATION,
"map_status": map_status,
"map": map_obj,
"locked_datasets": locked_datasets,
"remote_datasets": remote_datasets,
"downloadable_datasets": downloadable_datasets,
"site": site_url,
},
)
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
map_obj = _resolve_map(
request,
mapid,
'base.download_resourcebase',
_PERMISSION_MSG_VIEW)
map_status = dict()
if request.method == 'POST':
def perm_filter(layer):
return request.user.has_perm(
'base.view_resourcebase',
obj=layer.get_self_resource())
mapJson = map_obj.json(perm_filter)
# we need to remove duplicate layers
j_map = json.loads(mapJson)
j_layers = j_map["layers"]
for j_layer in j_layers:
if j_layer["service"] is None:
j_layers.remove(j_layer)
continue
if (len([_l for _l in j_layers if _l == j_layer])) > 1:
j_layers.remove(j_layer)
mapJson = json.dumps(j_map)
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
# TODO the url needs to be verified on geoserver
url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
elif check_ogc_backend(qgis_server.BACKEND_PACKAGE):
url = urljoin(settings.SITEURL,
reverse("qgis_server:download-map", kwargs={'mapid': mapid}))
# qgis-server backend stop here, continue on qgis_server/views.py
return redirect(url)
# the path to geoserver backend continue here
resp, content = http_client.request(url, 'POST', body=mapJson)
status = int(resp.status_code)
if status == 200:
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
raise Exception(
'Could not start the download of %s. Error was: %s' %
(map_obj.title, content))
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in map_obj.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(alternate=lyr.name)
if not request.user.has_perm(
'download_resourcebase',
obj=ownable_layer.get_self_resource()):
locked_layers.append(lyr)
else:
# we need to add the layer only once
if len(
[_l for _l in downloadable_layers if _l.name == lyr.name]) == 0:
downloadable_layers.append(lyr)
site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL
return render(request, template, context={
"geoserver": ogc_server_settings.PUBLIC_LOCATION,
"map_status": map_status,
"map": map_obj,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"site": site_url
})
def _render_thumbnail(self, spec):
resp, content = http_client.request(spec)
if resp.status < 200 or resp.status > 299:
logger.warning('Unable to obtain thumbnail: %s', content)
return
return content
def map_download(request, mapid, template='maps/map_download.html'):
"""
Download all the layers of a map as a batch
XXX To do, remove layer status once progress id done
This should be fix because
"""
map_obj = _resolve_map(request, mapid, 'base.download_resourcebase',
_PERMISSION_MSG_VIEW)
map_status = dict()
if request.method == 'POST':
def perm_filter(layer):
return request.user.has_perm('base.view_resourcebase',
obj=layer.get_self_resource())
mapJson = map_obj.json(perm_filter)
# we need to remove duplicate layers
j_map = json.loads(mapJson)
j_layers = j_map["layers"]
for j_layer in j_layers:
if j_layer["service"] is None:
j_layers.remove(j_layer)
continue
if (len([_l for _l in j_layers if _l == j_layer])) > 1:
j_layers.remove(j_layer)
mapJson = json.dumps(j_map)
if check_ogc_backend(qgis_server.BACKEND_PACKAGE):
url = urljoin(
settings.SITEURL,
reverse("qgis_server:download-map", kwargs={'mapid': mapid}))
# qgis-server backend stop here, continue on qgis_server/views.py
return redirect(url)
# the path to geoserver backend continue here
resp, content = http_client.request(url, 'POST', body=mapJson)
status = int(resp.status_code)
if status == 200:
map_status = json.loads(content)
request.session["map_status"] = map_status
else:
raise Exception(
'Could not start the download of %s. Error was: %s' %
(map_obj.title, content))
locked_layers = []
remote_layers = []
downloadable_layers = []
for lyr in map_obj.layer_set.all():
if lyr.group != "background":
if not lyr.local:
remote_layers.append(lyr)
else:
ownable_layer = Layer.objects.get(alternate=lyr.name)
if not request.user.has_perm(
'download_resourcebase',
obj=ownable_layer.get_self_resource()):
locked_layers.append(lyr)
else:
# we need to add the layer only once
if len([
_l for _l in downloadable_layers
if _l.name == lyr.name
]) == 0:
downloadable_layers.append(lyr)
site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith(
'http') else settings.SITEURL
register_event(request, EventType.EVENT_DOWNLOAD, map_obj)
return render(request,
template,
context={
"geoserver": ogc_server_settings.PUBLIC_LOCATION,
"map_status": map_status,
"map": map_obj,
"locked_layers": locked_layers,
"remote_layers": remote_layers,
"downloadable_layers": downloadable_layers,
"site": site_url
})
def proxy(request,
url=None,
response_callback=None,
sec_chk_hosts=True,
sec_chk_rules=True,
timeout=None,
**kwargs):
# Request default timeout
if not timeout:
timeout = TIMEOUT
# Security rules and settings
PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())
# Sanity url checks
if 'url' not in request.GET and not url:
return HttpResponse(
"The proxy service requires a URL-encoded URL as a parameter.",
status=400,
content_type="text/plain")
raw_url = url or request.GET['url']
raw_url = urljoin(settings.SITEURL,
raw_url) if raw_url.startswith("/") else raw_url
url = urlsplit(raw_url)
locator = str(url.path)
if url.query != "":
locator += '?' + url.query
if url.fragment != "":
locator += '#' + url.fragment
# White-Black Listing Hosts
if sec_chk_hosts and not settings.DEBUG:
site_url = urlsplit(settings.SITEURL)
if site_url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (site_url.hostname, )
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
from geonode.geoserver.helpers import ogc_server_settings
hostname = (
ogc_server_settings.hostname, ) if ogc_server_settings else ()
if hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += hostname
if url.query and ows_regexp.match(url.query):
ows_tokens = ows_regexp.match(url.query).groups()
if len(
ows_tokens
) == 4 and 'version' == ows_tokens[0] and StrictVersion(
ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion(
ows_tokens[1]
) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in (
'getcapabilities') and ows_tokens[3].upper() in (
'OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'):
if url.hostname not in PROXY_ALLOWED_HOSTS:
PROXY_ALLOWED_HOSTS += (url.hostname, )
if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
return HttpResponse(
"DEBUG is set to False but the host of the path provided to the proxy service"
" is not in the PROXY_ALLOWED_HOSTS setting.",
status=403,
content_type="text/plain")
# Security checks based on rules; allow only specific requests
if sec_chk_rules:
# TODO: Not yet implemented
pass
# Collecting headers and cookies
headers, access_token = get_headers(request, url, raw_url)
# Inject access_token if necessary
parsed = urlparse(raw_url)
parsed._replace(path=locator.encode('utf8'))
_url = parsed.geturl()
if request.method == "GET" and access_token and 'access_token' not in _url:
query_separator = '&' if '?' in _url else '?'
_url = ('%s%saccess_token=%s' % (_url, query_separator, access_token))
response, content = http_client.request(_url,
method=request.method,
data=request.body,
headers=headers,
timeout=timeout,
user=request.user)
content = response.content
status = response.status_code
content_type = response.headers.get('Content-Type')
# decompress GZipped responses if not enabled
# if content and response and response.getheader('Content-Encoding') == 'gzip':
if content and content_type and content_type == 'gzip':
from StringIO import StringIO
import gzip
buf = StringIO(content)
f = gzip.GzipFile(fileobj=buf)
content = f.read()
if response and response_callback:
kwargs = {} if not kwargs else kwargs
kwargs.update({
'response': response,
'content': content,
'status': status,
'content_type': content_type
})
return response_callback(**kwargs)
else:
# If we get a redirect, let's add a useful message.
if status and status in (301, 302, 303, 307):
_response = HttpResponse(
('This proxy does not support redirects. The server in "%s" '
'asked for a redirect to "%s"' %
(url, response.getheader('Location'))),
status=status,
content_type=content_type)
_response['Location'] = response.getheader('Location')
return _response
else:
return HttpResponse(content=content,
status=status,
content_type=content_type)
def geoserver_post_save_map(instance, sender, **kwargs):
instance.set_missing_info()
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(
Layer.objects.get(typename=layer.name).typename)
image = None
# If the map does not have any local layers, do not create the thumbnail.
if len(local_layers) > 0:
params = {
'layers': ",".join(local_layers).encode('utf-8'),
'format': 'image/png8',
'width': 200,
'height': 150,
}
# Add the bbox param only if the bbox is different to [None, None, None, None]
if None not in instance.bbox:
params['bbox'] = instance.bbox_string
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_remote_url = ogc_server_settings.LOCATION + "wms/reflect?" + p
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_remote_url,
defaults=dict(
extension='png',
name=_("Remote Thumbnail"),
mime='image/png',
link_type='image',
))
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbnail_remote_url)
if 'ServiceException' in image or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
if instance.has_thumbnail():
instance.thumbnail.thumb_file.delete()
else:
instance.thumbnail = Thumbnail()
instance.thumbnail.thumb_file.save('map-%s-thumb.png' % instance.id,
ContentFile(image))
instance.thumbnail.thumb_spec = thumbnail_remote_url
instance.thumbnail.save()
thumbnail_url = urljoin(settings.SITEURL,
instance.thumbnail.thumb_file.url)
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(
name=_('Thumbnail'),
extension='png',
mime='image/png',
link_type='image',
))
def get_gs_thumbnail(instance):
from geonode.geoserver.helpers import ogc_server_settings
if instance.class_name == 'Map':
local_layers = []
for layer in instance.layers:
if layer.local:
local_layers.append(layer.name)
layers = ",".join(local_layers).encode('utf-8')
if (len(local_layers) == 0):
return None
else:
layers = instance.typename.encode('utf-8')
logger.debug('Instance storeType: %s', instance.storeType)
params = {
'layers': layers,
'format': 'image/png8',
'width': 200,
'height': 150,
'TIME': '-99999999999-01-01T00:00:00.0Z/99999999999-01-01T00:00:00.0Z'
}
baseurl = ogc_server_settings.LOCATION + \
"wms/reflect?"
if (instance.storeType == 'remoteStore'):
params['request'] = 'getMap'
params['service'] = 'wms'
params['version'] = '1.3.0'
params['bbox'] = '%s,%s,%s,%s' % (instance.bbox_x0, instance.bbox_y0,
instance.bbox_x1, instance.bbox_y1)
baseurl = instance.ows_url + '?'
del params['TIME']
if all(s in baseurl.lower() for s in ['mapserver', 'wmsserver']):
params['styles'] = ''
params['crs'] = 'epsg:4326'
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_create_url = baseurl + p
if (instance.storeType == 'remoteStore'):
if (instance.service.type == 'REST'):
thumbnail_create_url = "%s/info/thumbnail" % (instance.ows_url)
tries = 0
max_tries = 15
while tries < max_tries:
logger.debug(
'Thumbnail: Requesting thumbnail from GeoServer. '
'Attempt %d of %d for %s', tries + 1, max_tries,
thumbnail_create_url)
if (instance.storeType == 'remoteStore'):
if tries > 4:
thumbnail_create_url = thumbnail_create_url.replace(
'image/png8', 'image/jpeg')
resp, image = http_client.request(thumbnail_create_url)
else:
# Login using basic auth as geoserver admin
user = settings.GEOSERVER_USER
pword = settings.GEOSERVER_PASSWORD
auth = base64.encodestring(user + ':' + pword)
resp, image = http_client.request(
thumbnail_create_url,
'GET',
headers={'Authorization': 'Basic ' + auth})
if 200 <= resp.status <= 299:
if 'ServiceException' not in image:
return image
else:
logger.debug(
'Thumbnail: Encountered unexpected status code: %d. '
'Aborting.', resp.status)
logger.debug(resp)
else:
# Unexpected Error Code, Stop Trying
logger.debug(
'Thumbnail: Encountered unexpected status code: %d. '
'Aborting.', resp.status)
logger.debug(resp)
break
# Layer not ready yet, try again
tries += 1
time.sleep(3)
return None
def _render_thumbnail(self, spec):
resp, content = http_client.request(spec)
if 'ServiceException' in content or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % content
raise RuntimeError(msg)
return content
def _enrich_layer_metadata(self, geonode_layer):
workspace, layername = geonode_layer.name.split(
":") if ":" in geonode_layer.name else (None, geonode_layer.name)
url = urlsplit(self.url)
base_url = '%s://%s/' % (url.scheme, url.netloc)
response = requests.get('%sapi/layers/?name=%s' %
(base_url, layername), {},
timeout=10,
verify=False)
content = response.content
status = response.status_code
content_type = response.headers['Content-Type']
if status == 200 and 'application/json' == content_type:
try:
if isinstance(content, bytes):
content = content.decode('UTF-8')
_json_obj = json.loads(content)
if _json_obj['meta']['total_count'] == 1:
_layer = _json_obj['objects'][0]
if _layer:
r_fields = {}
# Update plain fields
for field in GeoNodeServiceHandler.LAYER_FIELDS:
if field in _layer and _layer[field]:
r_fields[field] = _layer[field]
if r_fields:
Layer.objects.filter(id=geonode_layer.id).update(
**r_fields)
geonode_layer.refresh_from_db()
# Update Thumbnail
if "thumbnail_url" in _layer and _layer[
"thumbnail_url"]:
thumbnail_remote_url = _layer["thumbnail_url"]
_url = urlsplit(thumbnail_remote_url)
if not _url.scheme:
thumbnail_remote_url = "{}{}".format(
geonode_layer.remote_service.service_url,
_url.path)
resp, image = http_client.request(
thumbnail_remote_url)
if 'ServiceException' in str(image) or \
resp.status_code < 200 or resp.status_code > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid
geonode_layer.save_thumbnail(thumbnail_name,
image=image)
else:
self._create_layer_thumbnail(geonode_layer)
else:
self._create_layer_thumbnail(geonode_layer)
# Add Keywords
if "keywords" in _layer and _layer["keywords"]:
keywords = _layer["keywords"]
if keywords:
geonode_layer.keywords.clear()
geonode_layer.keywords.add(*keywords)
# Add Regions
if "regions" in _layer and _layer["regions"]:
(regions_resolved,
regions_unresolved) = resolve_regions(
_layer["regions"])
if regions_resolved:
geonode_layer.regions.clear()
geonode_layer.regions.add(*regions_resolved)
# Add Topic Category
if "category__gn_description" in _layer and _layer[
"category__gn_description"]:
try:
categories = TopicCategory.objects.filter(
Q(gn_description__iexact=_layer[
"category__gn_description"]))
if categories:
geonode_layer.category = categories[0]
except Exception:
traceback.print_exc()
except Exception:
traceback.print_exc()
finally:
try:
geonode_layer.save(notify=True)
except Exception as e:
logger.error(e)
def fetch(self, bbox: typing.List, zoom: int = None, *args, **kwargs):
"""
The function fetching tiles from a Slippy Map provider, composing them into a single image, and cropping it
to match the given BBOX. Retrieval of each tile is repeated self.max_retries times, waiting self.retry_delay
seconds between consecutive requests.
:param bbox: bounding box of the background image, dataset compliant format: [west, east, south, north, CRS]
:param zoom: zoom with which to retrieve Slippy Map's tiles (by default, it's calculated based on width, height)
:return: None if the CRS is different from self.tiles_crs, or background Image
"""
if not self.url:
logger.error("Thumbnail background requires url to be configured.")
raise ThumbnailError("Tiled background improperly configured.")
if bbox[-1].lower() != self.crs.lower():
# background service is not available the requested CRS CRS
logger.debug(
f"Thumbnail background generation skipped. "
f"Clashing CRSs: requested {bbox[-1]}, supported {self.crs}")
return
bbox = [float(coord) for coord in bbox[0:4]]
# check if BBOX fits within the EPSG:3857 map, if not - return an empty background
if bbox[2] > self._epsg3857_max_y or bbox[3] < -self._epsg3857_max_y:
return Image.new("RGB",
(self.thumbnail_width, self.thumbnail_height),
(250, 250, 250))
bbox4326 = self.bbox3857to4326(*bbox)
# change bbox from dataset (left, right, bottom, top) to mercantile (left, bottom, right, top)
self._mercantile_bbox = [
bbox4326[0], bbox4326[2], bbox4326[1], bbox4326[3]
]
# calculate zoom level
if zoom is None:
zoom = self.calculate_zoom()
else:
zoom = int(zoom)
top_left_tile = mercantile.tile(bbox4326[0], bbox4326[3], zoom)
bottom_right_tile = mercantile.tile(bbox4326[1], bbox4326[2], zoom)
# rescaling factors - indicators of how west and east BBOX boundaries are offset in respect to the world's map;
# east and west boundaries may exceed the maximum coordinate of the world in EPSG:3857. In such case additinal
# number of tiles need to be fetched to compose the image and the boundary tiles' coordinates need to be
# rescaled to ensure the proper image cropping.
epsg3857_world_width = 2 * self._epsg3857_max_x
west_rescaling_factor = 0
if abs(bbox[0]) > self._epsg3857_max_x:
west_rescaling_factor = ceil(
(abs(bbox[0]) - self._epsg3857_max_x) /
epsg3857_world_width) * copysign(1, bbox[0])
east_rescaling_factor = 0
if abs(bbox[1]) > self._epsg3857_max_x:
east_rescaling_factor = ceil(
(abs(bbox[1]) - self._epsg3857_max_x) /
epsg3857_world_width) * copysign(1, bbox[1])
map_row_tiles = 2**zoom - 1 # number of tiles in the Map's row for a certain zoom level
map_worlds = int(east_rescaling_factor -
west_rescaling_factor) # number maps in an image
worlds_between = map_worlds - 1 # number of full maps in an image
if top_left_tile.x > bottom_right_tile.x or bbox[1] - bbox[
0] > epsg3857_world_width or map_worlds > 0:
# BBOX crosses Slippy Map's border
if worlds_between > 0:
tiles_rows = (list(range(top_left_tile.x, map_row_tiles + 1)) +
worlds_between * list(range(map_row_tiles + 1)) +
list(range(bottom_right_tile.x + 1)))
else:
tiles_rows = list(range(top_left_tile.x,
map_row_tiles + 1)) + list(
range(bottom_right_tile.x + 1))
else:
# BBOx is contained by the Slippy Map
if worlds_between > 0:
tiles_rows = list(
range(top_left_tile.x, bottom_right_tile.x +
1)) + worlds_between * list(range(map_row_tiles + 1))
else:
tiles_rows = list(
range(top_left_tile.x, bottom_right_tile.x + 1))
tiles_cols = list(range(top_left_tile.y, bottom_right_tile.y + 1))
# if latitude boundaries extend world's height - add background's height, and set constant Y offset for tiles
additional_height = 0
fixed_top_offset = 0
fixed_bottom_offset = 0
north_extension3857 = max(0, bbox[3] - self._epsg3857_max_y)
south_extension3857 = abs(min(0, bbox[2] + self._epsg3857_max_y))
extension3857 = north_extension3857 + south_extension3857
if extension3857:
# get single tile's height in ESPG:3857
tile_bounds = mercantile.bounds(tiles_rows[0], tiles_cols[0], zoom)
_, south = self.point4326to3857(getattr(tile_bounds, "west"),
getattr(tile_bounds, "south"))
_, north = self.point4326to3857(getattr(tile_bounds, "west"),
getattr(tile_bounds, "north"))
tile_hight3857 = north - south
additional_height = round(
self.tile_size * extension3857 /
tile_hight3857) # based on linear proportion
if north_extension3857:
fixed_top_offset = round(self.tile_size * north_extension3857 /
tile_hight3857)
if south_extension3857:
fixed_bottom_offset = round(
self.tile_size * south_extension3857 / tile_hight3857)
background = Image.new(
"RGB",
(len(tiles_rows) * self.tile_size,
len(tiles_cols) * self.tile_size + additional_height),
(250, 250, 250),
)
for offset_x, x in enumerate(tiles_rows):
for offset_y, y in enumerate(tiles_cols):
if self.tms:
y = (2**zoom) - y - 1
imgurl = self.url.format(x=x, y=y, z=zoom)
im = None
for retries in range(self.max_retries):
try:
resp, content = http_client.request(imgurl)
if resp.status_code > 400:
retries = self.max_retries - 1
raise Exception(f"{strip_tags(content)}")
im = BytesIO(content)
Image.open(
im).verify() # verify that it is, in fact an image
break
except Exception as e:
logger.error(
f"Thumbnail background fetching from {imgurl} failed {retries} time(s) with: {e}"
)
if retries + 1 == self.max_retries:
raise e
time.sleep(self.retry_delay)
continue
if im:
image = Image.open(
im
) # "re-open" the file (required after running verify method)
# add the fetched tile to the background image, placing it under proper coordinates
background.paste(
image, (offset_x * self.tile_size,
offset_y * self.tile_size + fixed_top_offset))
# get BBOX of the tiles
top_left_bounds = mercantile.bounds(top_left_tile)
bottom_right_bounds = mercantile.bounds(bottom_right_tile)
tiles_bbox3857 = self.bbox4326to3857(
getattr(top_left_bounds, "west"),
getattr(bottom_right_bounds, "east"),
getattr(bottom_right_bounds, "south"),
getattr(top_left_bounds, "north"),
)
# rescale tiles' boundaries - if space covered by the input BBOX extends the width of the world,
# (e.g. two "worlds" are present on the map), translation between tiles' BBOX and image's pixel requires
# additional rescaling, for tiles' BBOX coordinates to match input BBOX coordinates
west_coord = tiles_bbox3857[
0] + west_rescaling_factor * epsg3857_world_width
east_coord = tiles_bbox3857[
1] + east_rescaling_factor * epsg3857_world_width
# prepare translating function from received BBOX to pixel values of the background image
src_quad = (0, fixed_top_offset, background.size[0],
background.size[1] - fixed_bottom_offset)
to_src_px = utils.make_bbox_to_pixels_transf(
[west_coord, tiles_bbox3857[2], east_coord, tiles_bbox3857[3]],
src_quad)
# translate received BBOX to pixel values
minx, miny = to_src_px(bbox[0], bbox[2])
maxx, maxy = to_src_px(bbox[1], bbox[3])
# max and min function for Y axis were introduced to mitigate rounding errors
crop_box = (
ceil(minx),
max(ceil(maxy) + fixed_top_offset, 0),
floor(maxx),
min(floor(miny) + fixed_top_offset, background.size[1]),
)
if not all([
0 <= crop_x <= background.size[0]
for crop_x in [crop_box[0], crop_box[2]]
]):
raise ThumbnailError(
f"Tiled background cropping error. Boundaries outside of the image: {crop_box}"
)
# crop background image to the desired bbox and resize it
background = background.crop(box=crop_box)
background = background.resize(
(self.thumbnail_width, self.thumbnail_height))
if sum(background.convert("L").getextrema()) in (0, 2):
# either all black or all white
logger.error("Thumbnail background outside the allowed area.")
raise ThumbnailError(
"Thumbnail background outside the allowed area.")
return background
def geoserver_post_save(instance, sender, **kwargs):
"""Save keywords to GeoServer
The way keywords are implemented requires the layer
to be saved to the database before accessing them.
"""
url = ogc_server_settings.internal_rest
try:
gs_resource = gs_catalog.get_resource(instance.name)
except (FailedRequestError, EnvironmentError) as e:
msg = ('Could not connect to geoserver at "%s"'
'to save information for layer "%s"' %
(ogc_server_settings.LOCATION, instance.name.encode('utf-8')))
logger.warn(msg, e)
# If geoserver is not online, there is no need to continue
return
# If there is no resource returned it could mean one of two things:
# a) There is a synchronization problem in geoserver
# b) The unit tests are running and another geoserver is running in the
# background.
# For both cases it is sensible to stop processing the layer
if gs_resource is None:
logger.warn('Could not get geoserver resource for %s' % instance)
return
gs_resource.keywords = instance.keyword_list()
#gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_resource)
bbox = gs_resource.latlon_bbox
dx = float(bbox[1]) - float(bbox[0])
dy = float(bbox[3]) - float(bbox[2])
dataAspect = 1 if dy == 0 else dx / dy
height = 550
width = int(height * dataAspect)
# Set download links for WMS, WCS or WFS and KML
links = wms_links(ogc_server_settings.public_url + 'wms?',
instance.typename.encode('utf-8'), instance.bbox_string,
instance.srid, height, width)
for ext, name, mime, wms_url in links:
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
name=ugettext(name),
defaults=dict(
extension=ext,
url=wms_url,
mime=mime,
link_type='image',
))
if instance.storeType == "dataStore":
links = wfs_links(ogc_server_settings.public_url + 'wfs?',
instance.typename.encode('utf-8'))
for ext, name, mime, wfs_url in links:
if mime == 'SHAPE-ZIP':
name = 'Zipped Shapefile'
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=wfs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
url=wfs_url,
link_type='data',
))
elif instance.storeType == 'coverageStore':
#FIXME(Ariel): This works for public layers, does it work for restricted too?
# would those end up with no geotiff links, like, forever?
permissions = {}
permissions['anonymous'] = instance.get_gen_level(ANONYMOUS_USERS)
permissions['authenticated'] = instance.get_gen_level(
AUTHENTICATED_USERS)
instance.set_gen_level(ANONYMOUS_USERS, 'layer_readonly')
#Potentially 3 dimensions can be returned by the grid if there is a z
#axis. Since we only want width/height, slice to the second dimension
covWidth, covHeight = get_coverage_grid_extent(instance)[:2]
links = wcs_links(ogc_server_settings.public_url + 'wcs?',
instance.typename.encode('utf-8'),
bbox=gs_resource.native_bbox[:-1],
crs=gs_resource.native_bbox[-1],
height=str(covHeight),
width=str(covWidth))
for ext, name, mime, wcs_url in links:
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=wcs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
link_type='data',
))
instance.set_gen_level(ANONYMOUS_USERS, permissions['anonymous'])
instance.set_gen_level(AUTHENTICATED_USERS,
permissions['authenticated'])
kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode(
{
'layers': instance.typename.encode('utf-8'),
'mode': "download"
})
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=kml_reflector_link_download,
defaults=dict(
extension='kml',
name=_("KML"),
mime='text/xml',
link_type='data',
))
kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode(
{
'layers': instance.typename.encode('utf-8'),
'mode': "refresh"
})
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=kml_reflector_link_view,
defaults=dict(
extension='kml',
name="View in Google Earth",
mime='text/xml',
link_type='data',
))
tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url +
'layers=%s' % instance.typename.encode('utf-8') +
'&zoom={z}&x={x}&y={y}' + '&format=image/png8')
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=tile_url,
defaults=dict(
extension='tiles',
name=_("Tiles"),
mime='image/png',
link_type='image',
))
wms_path = '%s/%s/wms' % (instance.workspace, instance.name)
ows_url = urljoin(ogc_server_settings.public_url, wms_path)
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=ows_url,
defaults=dict(
extension='html',
name=_("OWS"),
url=ows_url,
mime='text/html',
link_type='OGC:WMS',
))
html_link_url = '%s%s' % (settings.SITEURL[:-1],
instance.get_absolute_url())
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=html_link_url,
defaults=dict(
extension='html',
name=instance.typename,
mime='text/html',
link_type='html',
))
params = {
'layers': instance.typename.encode('utf-8'),
'format': 'image/png8',
'width': 200,
'height': 150,
}
# Avoid using urllib.urlencode here because it breaks the url.
# commas and slashes in values get encoded and then cause trouble
# with the WMS parser.
p = "&".join("%s=%s" % item for item in params.items())
thumbnail_url = ogc_server_settings.LOCATION + "wms/reflect?" + p
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=thumbnail_url,
defaults=dict(
extension='png',
name=_("Remote Thumbnail"),
mime='image/png',
link_type='image',
))
# Download thumbnail and save it locally.
resp, image = http_client.request(thumbnail_url)
if 'ServiceException' in image or resp.status < 200 or resp.status > 299:
msg = 'Unable to obtain thumbnail: %s' % image
logger.debug(msg)
# Replace error message with None.
image = None
if image is not None:
#Clean any orphan Thumbnail before
Thumbnail.objects.filter(resourcebase__id=None).delete()
thumbnail, created = Thumbnail.objects.get_or_create(
resourcebase__id=instance.id)
thumbnail.thumb_spec = thumbnail_url
thumbnail.save_thumb(image, instance._thumbnail_path())
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=settings.SITEURL +
instance._thumbnail_path(),
defaults=dict(
extension='png',
name=_("Thumbnail"),
mime='image/png',
link_type='image',
))
ogc_wms_url = ogc_server_settings.public_url + 'wms?'
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=ogc_wms_url,
defaults=dict(
extension='html',
name=instance.name,
url=ogc_wms_url,
mime='text/html',
link_type='OGC:WMS',
))
if instance.storeType == "dataStore":
ogc_wfs_url = ogc_server_settings.public_url + 'wfs?'
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=ogc_wfs_url,
defaults=dict(
extension='html',
name=instance.name,
url=ogc_wfs_url,
mime='text/html',
link_type='OGC:WFS',
))
if instance.storeType == "coverageStore":
ogc_wcs_url = ogc_server_settings.public_url + 'wcs?'
Link.objects.get_or_create(resource=instance.resourcebase_ptr,
url=ogc_wcs_url,
defaults=dict(
extension='html',
name=instance.name,
url=ogc_wcs_url,
mime='text/html',
link_type='OGC:WCS',
))
#remove links that belong to and old address
for link in instance.link_set.all():
if not urlparse(settings.SITEURL).hostname == urlparse(link.url).hostname and not \
urlparse(ogc_server_settings.public_url).hostname == urlparse(link.url).hostname:
link.delete()
#Save layer attributes
set_attributes(instance)
#Save layer styles
set_styles(instance, gs_catalog)
