def get_layer_capabilities(layer,
version='1.3.0',
access_token=None,
tolerant=False):
"""
Retrieve a layer-specific GetCapabilities document
"""
workspace, layername = layer.alternate.split(
":") if ":" in layer.alternate else (None, layer.alternate)
if not layer.remote_service:
wms_url = f'{ogc_server_settings.LOCATION}{workspace}/{layername}/wms?service=wms&version={version}&request=GetCapabilities' # noqa
if access_token:
wms_url += f'&access_token={access_token}'
else:
wms_url = f'{layer.remote_service.service_url}?service=wms&version={version}&request=GetCapabilities'
_user, _password = ogc_server_settings.credentials
req, content = http_client.get(wms_url, user=_user)
getcap = ensure_string(content)
if not getattr(settings, 'DELAYED_SECURITY_SIGNALS', False):
if tolerant and ('ServiceException' in getcap
or req.status_code == 404):
# WARNING Please make sure to have enabled DJANGO CACHE as per
# https://docs.djangoproject.com/en/2.0/topics/cache/#filesystem-caching
wms_url = f'{ogc_server_settings.public_url}{workspace}/ows?service=wms&version={version}&request=GetCapabilities&layers={layer}' # noqa
if access_token:
wms_url += f'&access_token={access_token}'
req, content = http_client.get(wms_url, user=_user)
getcap = ensure_string(content)
if 'ServiceException' in getcap or req.status_code == 404:
return None
return getcap.encode('UTF-8')
def test_list_style(self):
"""Test querying list of styles from QGIS Server."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
layer = file_upload(filename)
""":type: geonode.layers.models.Layer"""
actual_list_style = style_list(layer, internal=False)
expected_list_style = ['default']
# There will be a default style
if actual_list_style:
self.assertEqual(
set(expected_list_style),
set([style.name for style in actual_list_style]))
style_list_url = reverse(
'qgis_server:download-qml',
kwargs={
'layername': layer.name
})
response = self.client.get(style_list_url)
self.assertEqual(response.status_code, 200)
content = ensure_string(response.content)
if isinstance(content, bytes):
content = content.decode('UTF-8')
actual_list_style = json.loads(content)
# There will be a default style
self.assertEqual(
set(expected_list_style),
set([style['name'] for style in actual_list_style]))
layer.delete()
def layer_ogc_request(request, layername):
"""Provide one OGC server per layer, with their own GetCapabilities.
:param layername: The layer name in Geonode.
:type layername: basestring
:return: The HTTPResponse with the response from QGIS Server.
"""
layer = get_object_or_404(Layer, name=layername)
qgis_layer = get_object_or_404(QGISServerLayer, layer=layer)
params = {
'MAP': qgis_layer.qgis_project_path,
}
params.update(request.GET or request.POST)
response = requests.get(QGIS_SERVER_CONFIG['qgis_server_url'], params)
# We need to replace every references to the internal QGIS Server IP to
# the public Geonode URL.
public_url = requests.compat.urljoin(
settings.SITEURL,
reverse('qgis_server:layer-request', kwargs={'layername': layername}))
is_text = response.headers.get('content-type').startswith('text')
raw = ensure_string(response.content)
if is_text:
raw = raw.replace(QGIS_SERVER_CONFIG['qgis_server_url'], public_url)
return HttpResponse(raw, content_type=response.headers.get('content-type'))
def test_access_document_upload_form(self):
"""Test the form page is returned correctly via GET request /documents/upload"""
log = self.client.login(username='******', password='******')
self.assertTrue(log)
response = self.client.get(reverse('document_upload'))
self.assertTrue('Upload Documents' in ensure_string(response.content))
def test_data_json(self):
"""Test that the data.json representation behaves correctly"""
response = self.client.get(reverse('data_json')).content
data_json = json.loads(ensure_string(response))
len1 = len(ResourceBase.objects.all())
len2 = len(data_json)
self.assertEqual(len1, len2,
'Expected equality of json and repository lengths')
record_keys = [
u'publisher',
u'title',
u'description',
u'mbox',
u'keyword',
u'modified',
u'contactPoint',
u'accessLevel',
u'distribution',
u'identifier',
]
for record in data_json:
self.assertEqual(sorted(record_keys), sorted(list(record.keys())),
'Expected specific list of fields to output')
def test_ajax_document_permissions(self):
"""Verify that the ajax_document_permissions view is behaving as expected
"""
# Setup some document names to work with
f = SimpleUploadedFile(
'test_img_file.gif',
self.imgfile.read(),
'image/gif')
superuser = get_user_model().objects.get(pk=2)
document = Document.objects.create(
doc_file=f,
owner=superuser,
title='theimg')
document.set_default_permissions()
document_id = document.id
invalid_document_id = 20
# Test that an invalid document is handled for properly
response = self.client.post(
reverse(
'resource_permissions', args=(
invalid_document_id,)), data=json.dumps(
self.perm_spec), content_type="application/json")
self.assertEqual(response.status_code, 404)
# Test that GET returns permissions
response = self.client.get(reverse('resource_permissions', args=(document_id,)))
assert('permissions' in ensure_string(response.content))
# Test that a user is required to have
# documents.change_layer_permissions
# First test un-authenticated
response = self.client.post(
reverse('resource_permissions', args=(document_id,)),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Next Test with a user that does NOT have the proper perms
logged_in = self.client.login(username='******', password='******')
self.assertEqual(logged_in, True)
response = self.client.post(
reverse('resource_permissions', args=(document_id,)),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Login as a user with the proper permission and test the endpoint
logged_in = self.client.login(username='******', password='******')
self.assertEqual(logged_in, True)
response = self.client.post(
reverse('resource_permissions', args=(document_id,)),
data=json.dumps(self.perm_spec),
content_type="application/json")
# Test that the method returns 200
self.assertEqual(response.status_code, 200)
def test_ajax_document_permissions(self, create_thumb):
"""Verify that the ajax_document_permissions view is behaving as expected
"""
create_thumb.return_value = True
# Setup some document names to work with
f = [f"{settings.MEDIA_ROOT}/img.gif"]
superuser = get_user_model().objects.get(pk=2)
document = resource_manager.create(None,
resource_type=Document,
defaults=dict(files=f,
owner=superuser,
title='theimg',
is_approved=True))
document_id = document.id
invalid_document_id = 20
# Test that an invalid document is handled for properly
response = self.client.post(reverse('resource_permissions',
args=(invalid_document_id, )),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Test that GET returns permissions
response = self.client.get(
reverse('resource_permissions', args=(document_id, )))
assert ('permissions' in ensure_string(response.content))
# Test that a user is required to have
# documents.change_dataset_permissions
# First test un-authenticated
response = self.client.post(reverse('resource_permissions',
args=(document_id, )),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Next Test with a user that does NOT have the proper perms
logged_in = self.client.login(username='******', password='******')
self.assertEqual(logged_in, True)
response = self.client.post(reverse('resource_permissions',
args=(document_id, )),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Login as a user with the proper permission and test the endpoint
logged_in = self.client.login(username='******', password='******')
self.assertEqual(logged_in, True)
response = self.client.post(reverse('resource_permissions',
args=(document_id, )),
data=json.dumps(self.perm_spec),
content_type="application/json")
# Test that the method returns 200
self.assertEqual(response.status_code, 200)
def test_ajax_map_permissions(self):
"""Verify that the ajax_layer_permissions view is behaving as expected
"""
# Setup some layer names to work with
mapid = Map.objects.all().first().pk
invalid_mapid = "42"
def url(id):
return reverse('resource_permissions', args=[id])
# Test that an invalid layer.alternate is handled for properly
response = self.client.post(
url(invalid_mapid),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 404)
# Test that GET returns permissions
response = self.client.get(url(mapid))
assert('permissions' in ensure_string(response.content))
# Test that a user is required to have permissions
# First test un-authenticated
response = self.client.post(
url(mapid),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Next Test with a user that does NOT have the proper perms
logged_in = self.client.login(username='******', password='******')
self.assertEqual(logged_in, True)
response = self.client.post(
url(mapid),
data=json.dumps(self.perm_spec),
content_type="application/json")
self.assertEqual(response.status_code, 401)
# Login as a user with the proper permission and test the endpoint
logged_in = self.client.login(username='******', password='******')
self.assertEqual(logged_in, True)
response = self.client.post(
url(mapid),
data=json.dumps(self.perm_spec),
content_type="application/json")
# Test that the method returns 200
self.assertEqual(response.status_code, 200)
def test_tile_url(self):
"""Test to return tile format."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
tile_format = tile_url_format(uploaded.name)
# Accessing this directly should return 404
response = self.client.get(tile_format)
self.assertEqual(response.status_code, 404)
qgis_tile_url = tile_url(uploaded, 11, 1576, 1054, internal=True)
parse_result = urlparse(qgis_tile_url)
base_net_loc = urlparse(settings.QGIS_SERVER_URL).netloc
self.assertEqual(base_net_loc, parse_result.netloc)
query_string = parse_qs(parse_result.query)
expected_query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetMap',
'BBOX': '10801469.341,-606604.256471,10821037.2203,-587036.37723',
'CRS': 'EPSG:3857',
'WIDTH': '256',
'HEIGHT': '256',
'LAYERS': 'test_grid',
'STYLE': 'default',
'FORMAT': 'image/png',
'TRANSPARENT': 'true',
'DPI': '96',
'MAP_RESOLUTION': '96',
'FORMAT_OPTIONS': 'dpi:96'
}
for key, value in expected_query_string.items():
# urlparse.parse_qs returned a dictionary of list
actual_value = query_string[key][0]
self.assertEqual(actual_value, value)
# Check that qgis server returns valid url
# Use python requests because the endpoint is not django
response = requests.get(qgis_tile_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
uploaded.delete()
def test_thumbnail_links(self):
"""Test that thumbnail links were created after upload."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
layer = file_upload(filename)
""":type: geonode.layers.models.Layer"""
# check that we have remote thumbnail
remote_thumbnail_link = layer.link_set.get(
name__icontains='remote thumbnail')
self.assertTrue(remote_thumbnail_link.url)
# thumbnail won't generate because remote thumbnail uses public
# address
remote_thumbnail_url = remote_thumbnail_link.url
# Replace url's basename, we want to access it using django client
parse_result = urlsplit(remote_thumbnail_url)
remote_thumbnail_url = urlunsplit(
('', '', parse_result.path, parse_result.query, ''))
response = self.client.get(remote_thumbnail_url)
thumbnail_dir = os.path.join(settings.MEDIA_ROOT, 'thumbs')
thumbnail_path = os.path.join(thumbnail_dir, 'layer-thumb.png')
layer.save_thumbnail(thumbnail_path, ensure_string(response.content))
# Check thumbnail created
self.assertTrue(os.path.exists(thumbnail_path))
self.assertEqual(what(thumbnail_path), 'png')
# Check that now we have thumbnail
self.assertTrue(layer.has_thumbnail())
missing_thumbnail_url = staticfiles.static(settings.MISSING_THUMBNAIL)
self.assertTrue(layer.get_thumbnail_url() != missing_thumbnail_url)
thumbnail_links = layer.link_set.filter(name__icontains='thumbnail')
self.assertTrue(len(thumbnail_links) > 0)
link_names = ['remote thumbnail', 'thumbnail']
for link in thumbnail_links:
self.assertIn(link.name.lower(), link_names)
# cleanup
layer.delete()
def test_download_map_qlr(self):
"""Test download QLR file for a map"""
# 2 layers to be added to the map
filename = os.path.join(
gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
layer1 = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
layer2 = file_upload(filename)
# construct json request for new map
json_payload = InitialSetup.generate_initial_map(layer1, layer2)
self.client.login(username='******', password='******')
response = self.client.post(
reverse('new_map_json'),
data=json.dumps(json_payload),
content_type='application/json')
# map is successfully saved
self.assertEqual(response.status_code, 200)
content = ensure_string(response.content)
if isinstance(content, bytes):
content = content.decode('UTF-8')
map_id = json.loads(content).get('id')
map = Map.objects.get(id=map_id)
# check that QLR is added to the links
links = map.link_set.download()
map_qlr_link = links.get(name='Download QLR Layer file')
self.assertIn('qlr', map_qlr_link.url)
# QLR
response = self.client.get(
reverse('map_download_qlr', kwargs={'mapid': map_id}))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get('Content-Type'),
'application/x-qgis-layer-definition')
# cleanup
map.delete()
layer1.delete()
layer2.delete()
def cache_request(self, url, cache_file):
"""Cache a given url request to a file.
On some rare occasions, QGIS Server url request is taking too long to
complete. This could be a problem if user is requesting something like
a tile or legend from Web interface and when it takes too long, the
connection will reset. This case will make the request never be completed
because the request died with user connection.
For this kind of request, it is better to register the request as celery
task. This will make the task to keep running, even if connection is
reset.
:param url: The target url to request
:type url: str
:param cache_file: The target file path to save the cache
:type cache_file: str
:return: True if succeeded
:rtype: bool
"""
logger.debug('Requesting url: {url}'.format(url=url))
response = requests.get(url, stream=True)
if not response.status_code == 200:
# Failed to fetch request. Abort with error message
msg = (
'Failed to fetch requested url: {url}\n'
'With HTTP status code: {status_code}\n'
'Content: {content}')
msg = msg.format(
url=url,
status_code=response.status_code,
content=ensure_string(response.content))
raise HTTPError(msg)
with open(cache_file, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
del response
return True
def from_get_capabilities_style_xml(cls,
qgis_layer,
style_xml,
style_url=None,
synchronize=True):
"""Convert to this model from GetCapabilities Style tag.
:param qgis_layer: Associated QGIS Server Layer
:type qgis_layer: QGISServerLayer
:param style_xml: xml string or object
:type style_xml: str | lxml.etree.Element |
xml.etree.ElementTree.Element
:param style_url: style information stored as xml
:type style_url: str
:param synchronize: Flag, if true then synchronize the new value
:type synchronize: bool
:return: QGISServerStyle model and boolean flag created
:rtype: QGISServerStyle, bool
"""
if isinstance(style_xml, string_types):
style_xml = dlxml.fromstring(style_xml)
elif isinstance(style_xml, ElementTree.Element):
style_xml = dlxml.fromstring(
ElementTree.tostring(style_xml, encoding='utf-8',
method='xml'))
namespaces = {
'wms': 'http://www.opengis.net/wms',
'xlink': 'http://www.w3.org/1999/xlink'
}
filter_dict = {
'name': style_xml.xpath('wms:Name', namespaces=namespaces)[0].text,
'layer_styles': qgis_layer
}
# if style_body is none, try fetch it from QGIS Server
if not style_url:
from geonode.qgis_server.helpers import style_get_url
style_url = style_get_url(qgis_layer.layer,
filter_dict['name'],
internal=False)
response = requests.get(style_url)
style_body = etree.tostring(dlxml.fromstring(
ensure_string(response.content)),
pretty_print=True)
default_dict = {
'title':
style_xml.xpath('wms:Title', namespaces=namespaces)[0].text,
'style_legend_url':
style_xml.xpath('wms:LegendURL/wms:OnlineResource',
namespaces=namespaces)
[0].attrib['{http://www.w3.org/1999/xlink}href'],
'style_url':
style_url,
'body':
style_body
}
# filter_dict['defaults'] = default_dict
# Can't use get_or_create function for some reason.
# So use regular query
try:
style_obj = QGISServerStyle.objects.get(**filter_dict)
created = False
except QGISServerStyle.DoesNotExist:
style_obj = QGISServerStyle(**default_dict)
style_obj.name = filter_dict['name']
style_obj.save()
created = True
if created or synchronize:
# Try to synchronize this model with the given parameters
style_obj.name = filter_dict['name']
style_obj.style_url = default_dict['style_url']
style_obj.body = default_dict['body']
style_obj.title = default_dict['title']
style_obj.style_legend_url = default_dict['style_legend_url']
style_obj.save()
style_obj.layer_styles.add(qgis_layer)
style_obj.save()
return style_obj, created
def create_models(type=None, integration=False):
map_data, user_data, people_data, layer_data, document_data = create_fixtures(
)
anonymous_group, created = Group.objects.get_or_create(name='anonymous')
with transaction.atomic():
logger.info("[SetUp] Get or create user admin")
u = get_user_model().objects.filter(username='******').first()
if not u:
try:
u = get_user_model().objects.create(username='******',
is_superuser=True,
first_name='admin')
except Exception:
raise
if u:
u.set_password('admin')
u.save()
users = []
for ud, pd in zip(user_data, cycle(people_data)):
user_name, password, first_name, last_name = ud
with transaction.atomic():
try:
logger.info(f"[SetUp] Get or create user {user_name}")
u, created = get_user_model().objects.get_or_create(
username=user_name)
if created:
u.set_password(password)
u.first_name = first_name
u.last_name = last_name
u.save()
u.groups.add(anonymous_group)
users.append(u)
except Exception:
raise
logger.info(f"[SetUp] Add group {anonymous_group}")
get_user_model().objects.get(
username='******').groups.add(anonymous_group)
obj_ids = []
from geonode.utils import DisableDjangoSignals
with DisableDjangoSignals(skip=integration):
if not type or ensure_string(type) == 'map':
for md, user in zip(map_data, cycle(users)):
title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0,
bbox_y1), category = md
logger.info(f"[SetUp] Add map {title}")
m = Map(title=title,
abstract=abstract,
zoom=4,
projection='EPSG:4326',
center_x=42,
center_y=-73,
owner=user,
bbox_polygon=Polygon.from_bbox(
(bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
srid='EPSG:4326',
category=category)
m.save()
m.set_default_permissions()
obj_ids.append(m.id)
for kw in kws:
m.keywords.add(kw)
m.save()
if not type or ensure_string(type) == 'document':
for dd, user in zip(document_data, cycle(users)):
title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0,
bbox_y1), category = dd
logger.info(f"[SetUp] Add document {title}")
m = Document(title=title,
abstract=abstract,
owner=user,
bbox_polygon=Polygon.from_bbox(
(bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
srid='EPSG:4326',
category=category,
doc_file=f)
m.save()
m.set_default_permissions()
obj_ids.append(m.id)
for kw in kws:
m.keywords.add(kw)
m.save()
if not type or ensure_string(type) == 'layer':
for ld, owner, storeType in zip(
layer_data, cycle(users),
cycle(('coverageStore', 'dataStore'))):
title, abstract, name, alternate, (
bbox_x0, bbox_x1, bbox_y0,
bbox_y1), start, kws, category = ld
end = start + timedelta(days=365)
logger.info(f"[SetUp] Add layer {title}")
layer = Layer(title=title,
abstract=abstract,
name=name,
alternate=alternate,
bbox_polygon=Polygon.from_bbox(
(bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
srid='EPSG:4326',
uuid=str(uuid4()),
owner=owner,
temporal_extent_start=start,
temporal_extent_end=end,
date=start,
storeType=storeType,
category=category)
layer.save()
layer.set_default_permissions()
obj_ids.append(layer.id)
for kw in kws:
layer.keywords.add(kw)
layer.save()
return obj_ids
def test_style_management_url(self):
"""Test QGIS Server style management url construction."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
# Get default style
# There will always be a default style when uploading a layer
style_url = style_get_url(uploaded, 'default', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers.get('Content-Type'), 'text/xml')
# it has to contains qgis tags
style_xml = dlxml.fromstring(ensure_string(response.content))
self.assertTrue('qgis' in style_xml.tag)
# Add new style
# change default style slightly
self.assertTrue('WhiteToBlack' not in ensure_string(response.content))
self.assertTrue('BlackToWhite' in ensure_string(response.content))
new_style_xml = dlxml.fromstring(
ensure_string(response.content).replace('BlackToWhite', 'WhiteToBlack'))
new_xml_content = etree.tostring(new_style_xml, pretty_print=True)
# save it to qml file, accessible by qgis server
qgis_layer = QGISServerLayer.objects.get(layer=uploaded)
with open(qgis_layer.qml_path, mode='w') as f:
f.write(new_xml_content)
style_url = style_add_url(uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(ensure_string(response.content), 'OK')
# Get style list
qml_styles = style_list(uploaded, internal=False)
if qml_styles:
expected_style_names = ['default', 'new_style']
actual_style_names = [s.name for s in qml_styles]
self.assertEqual(
set(expected_style_names),
set(actual_style_names))
# Get new style
style_url = style_get_url(uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers.get('Content-Type'), 'text/xml')
self.assertTrue('WhiteToBlack' in ensure_string(response.content))
# Set default style
style_url = style_set_default_url(
uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(ensure_string(response.content), 'OK')
# Remove style
style_url = style_remove_url(uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(ensure_string(response.content), 'OK')
# Cleanup
uploaded.delete()
def update_from_viewer(self, conf, context=None):
"""
Update this Map's details by parsing a JSON object as produced by
a GXP Viewer.
This method automatically persists to the database!
"""
template_name = hookset.update_from_viewer(conf, context=context)
if not isinstance(context, dict):
try:
context = json.loads(ensure_string(context))
except Exception:
pass
conf = context.get("config", {})
if not isinstance(conf, dict) or isinstance(conf, bytes):
try:
conf = json.loads(ensure_string(conf))
except Exception:
conf = {}
about = conf.get("about", {})
self.title = conf.get("title", about.get("title", ""))
self.abstract = conf.get("abstract", about.get("abstract", ""))
_map = conf.get("map", {})
center = _map.get("center", settings.DEFAULT_MAP_CENTER)
self.zoom = _map.get("zoom", settings.DEFAULT_MAP_ZOOM)
if isinstance(center, dict):
self.center_x = center.get("x")
self.center_y = center.get("y")
else:
self.center_x, self.center_y = center
projection = _map.get("projection", settings.DEFAULT_MAP_CRS)
bbox = _map.get("bbox", None)
if bbox:
self.set_bounds_from_bbox(bbox, projection)
else:
self.set_bounds_from_center_and_zoom(self.center_x, self.center_y, self.zoom)
if self.projection is None or self.projection == "":
self.projection = projection
if self.uuid is None or self.uuid == "":
self.uuid = str(uuid.uuid1())
def source_for(layer):
try:
return conf["sources"][layer["source"]]
except Exception:
if "url" in layer:
return {"url": layer["url"]}
else:
return {}
layers = [lyr for lyr in _map.get("layers", [])]
dataset_names = {lyr.alternate for lyr in self.local_datasets}
self.dataset_set.all().delete()
self.keywords.add(*_map.get("keywords", []))
for ordering, layer in enumerate(layers):
self.dataset_set.add(dataset_from_viewer_config(self.id, MapLayer, layer, source_for(layer), ordering))
from geonode.resource.manager import resource_manager
resource_manager.update(self.uuid, instance=self, notify=True)
resource_manager.set_thumbnail(self.uuid, instance=self, overwrite=False)
if dataset_names != {lyr.alternate for lyr in self.local_datasets}:
map_changed_signal.send_robust(sender=self, what_changed="datasets")
return template_name
def style_list(layer, internal=True, generating_qgis_capabilities=False):
"""Query list of styles from QGIS Server.
:param layer: Layer to inspect
:type layer: Layer
:param internal: Flag to switch between public url and internal url.
Public url will be served by Django Geonode (proxified).
:type internal: bool
:param generating_qgis_capabilities: internal Flag for the method to tell
that this function were executed to generate QGIS GetCapabilities
request for querying Style list. This flag is used for recursion.
Default to False as recursion base.
:type generating_qgis_capabilities: bool
:return: List of QGISServerStyle
:rtype: list(QGISServerStyle)
"""
# We get the list of style from GetCapabilities request
# Must call from public URL because we need public LegendURL
url = wms_get_capabilities_url(layer, internal=internal)
try:
response = requests.get(url)
root_xml = dlxml.fromstring(ensure_string(response.content))
styles_xml = root_xml.xpath(
'wms:Capability/wms:Layer/wms:Layer/wms:Style',
namespaces={
'xlink': 'http://www.w3.org/1999/xlink',
'wms': 'http://www.opengis.net/wms'
})
# Fetch styles body
try:
qgis_layer = QGISServerLayer.objects.get(layer=layer)
except QGISServerLayer.DoesNotExist:
msg = 'No QGIS Server Layer for existing layer {0}'.format(
layer.name)
logger.debug(msg)
raise
styles_obj = [
QGISServerStyle.from_get_capabilities_style_xml(
qgis_layer, style_xml)[0] for style_xml in styles_xml
]
# Only tried to generate/fix QGIS GetCapabilities to return correct style
# list, if:
# - the current request return empty styles_obj (no styles, not possible)
# - does not currently tried to generate QGIS GetCapabilities to fix this
# problem
if not styles_obj and not generating_qgis_capabilities:
# It's not possible to have empty style. There will always be default
# style.
# Initiate a dummy requests to trigger build style list on QGIS Server
# side
# write an empty file if it doesn't exists
open(qgis_layer.qml_path, 'a').close()
# Basically add a new style then deletes it to force QGIS to refresh
# style list in project properties. We don't care the request result.
dummy_style_name = '__tmp__dummy__name__'
style_url = style_add_url(layer, dummy_style_name)
requests.get(style_url)
style_url = style_remove_url(layer, dummy_style_name)
requests.get(style_url)
# End the requests and rely on the next request to build style models
# to avoid infinite recursion
# Set generating_qgis_capabilities flag to True to avoid next
# recursion
return style_list(layer,
internal=internal,
generating_qgis_capabilities=True)
# Manage orphaned styles
style_names = [s.name for s in styles_obj]
for style in qgis_layer.styles.all():
if style.name not in style_names:
if style == qgis_layer.default_style:
qgis_layer.default_style = None
qgis_layer.save()
style.delete()
# Set default style if not yet set
set_default_style = False
try:
if not qgis_layer.default_style:
set_default_style = True
except BaseException:
set_default_style = True
if set_default_style and styles_obj:
qgis_layer.default_style = styles_obj[0]
qgis_layer.save()
return styles_obj
except BaseException:
msg = 'No QGIS Style for existing layer {0}'.format(layer.name)
logger.debug(msg)
raise
def test_unpublished(self):
"""Test permissions on an unpublished layer
"""
layer = Layer.objects.first()
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
# time.sleep(20)
try:
# request getCapabilities: layer must be there as it is published and
# advertised: we need to check if in response there is
# <Name>geonode:san_andres_y_providencia_water</Name>
geoserver_base_url = settings.OGC_SERVER['default']['LOCATION']
get_capabilities_url = 'ows?' \
'service=wms&version=1.3.0&request=GetCapabilities'
url = urljoin(geoserver_base_url, get_capabilities_url)
str_to_check = f'<Name>geonode:{layer.name}</Name>'
request = Request(url)
response = urlopen(request)
# by default the uploaded layer is published
self.assertTrue(layer.is_published)
self.assertTrue(any(str_to_check in ensure_string(s) for s in response.readlines()))
finally:
# Clean up and completely delete the layer
layer.delete()
# with settings disabled
with self.settings(RESOURCE_PUBLISHING=True):
layer = Layer.objects.first()
layer.is_approved = False
layer.is_published = False
layer.save()
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
try:
# by default the uploaded layer must be unpublished
self.assertEqual(layer.is_published, False)
# check the layer is not in GetCapabilities
request = Request(url)
response = urlopen(request)
# now test with published layer
layer = Layer.objects.get(pk=layer.pk)
layer.is_published = True
layer.save()
# we need some time to have the service up and running
time.sleep(20)
request = Request(url)
response = urlopen(request)
self.assertTrue(any(str_to_check in ensure_string(s) for s in response.readlines()))
finally:
# Clean up and completely delete the layer
layer.delete()
def test_ogc_specific_layer(self):
"""Test we can use QGIS Server API for a layer.
For now, we are just checking we can call these views without any
exceptions. We should improve this test by checking the result.
"""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
vector_layer = file_upload(filename)
params = {'layername': uploaded.name}
# Zip
response = self.client.get(
reverse('qgis_server:download-zip', kwargs=params))
self.assertEqual(response.status_code, 200)
try:
f = io.StringIO(ensure_string(response.content))
zipped_file = zipfile.ZipFile(f, 'r')
for one_file in zipped_file.namelist():
# We shoudn't get any QGIS project
self.assertFalse(one_file.endswith('.qgs'))
self.assertIsNone(zipped_file.testzip())
finally:
zipped_file.close()
f.close()
# Legend
response = self.client.get(
reverse('qgis_server:legend', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
# Tile
coordinates = {'z': '11', 'x': '1576', 'y': '1054'}
coordinates.update(params)
response = self.client.get(
reverse('qgis_server:tile', kwargs=coordinates))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
# Tile 404
response = self.client.get(
reverse('qgis_server:tile', kwargs=params))
self.assertEqual(response.status_code, 404)
self.assertEqual(
response.get('Content-Type'), 'text/html; charset=utf-8')
# Geotiff
response = self.client.get(
reverse('qgis_server:geotiff', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/tiff')
self.assertEqual(what('', h=ensure_string(response.content)), 'tiff')
# Layer is already on the database
# checking the Link
links = uploaded.link_set.download().filter(
name__in=settings.DOWNLOAD_FORMATS_RASTER)
# checks signals.py for the hardcoded names in QLR and QGS
qlr_link = links.get(name='QGIS layer file (.qlr)')
self.assertIn("download-qlr", qlr_link.url)
qgs_link = links.get(name='QGIS project file (.qgs)')
self.assertIn("download-qgs", qgs_link.url)
# QLR
response = self.client.get(
reverse('qgis_server:download-qlr', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get('Content-Type'),
'application/x-qgis-layer-definition')
# check file name's extension
file_name = response.get('Content-Disposition').split('filename=')
file_ext = file_name[1].split('.')
self.assertEqual(file_ext[1], "qlr")
# QGS
response = self.client.get(
reverse('qgis_server:download-qgs', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get('Content-Type'),
'application/x-qgis-project')
# check file name's extension
file_name = response.get('Content-Disposition').split('filename=')
file_ext = file_name[1].split('.')
self.assertEqual(file_ext[1], "qgs")
response = self.client.get(
reverse('qgis_server:geotiff', kwargs={
'layername': vector_layer.name
}))
self.assertEqual(response.status_code, 404)
# QML Styles
# Request list of styles
response = self.client.get(
reverse('qgis_server:download-qml', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'application/json')
# Should return a default style list
content = ensure_string(response.content)
if isinstance(content, bytes):
content = content.decode('UTF-8')
actual_result = json.loads(content)
actual_result = [s['name'] for s in actual_result]
expected_result = ['default']
self.assertEqual(set(expected_result), set(actual_result))
# Get single styles
response = self.client.get(
reverse('qgis_server:download-qml', kwargs={
'layername': params['layername'],
'style_name': 'default'
}))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'text/xml')
# Set thumbnail from viewed bbox
response = self.client.get(
reverse('qgis_server:set-thumbnail', kwargs=params))
self.assertEqual(response.status_code, 400)
data = {
'bbox': '-5.54025,96.9406,-5.2820,97.1250'
}
response = self.client.post(
reverse('qgis_server:set-thumbnail', kwargs=params),
data=data)
# User dont have permission
self.assertEqual(response.status_code, 403)
# Should log in
self.client.login(username='******', password='******')
response = self.client.post(
reverse('qgis_server:set-thumbnail', kwargs=params),
data=data)
self.assertEqual(response.status_code, 200)
content = ensure_string(response.content)
if isinstance(content, bytes):
content = content.decode('UTF-8')
retval = json.loads(content)
expected_retval = {
'success': True
}
self.assertEqual(retval, expected_retval)
# OGC Server specific for THE layer
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetLegendGraphics',
'FORMAT': 'image/png',
'LAYERS': uploaded.name,
}
response = self.client.get(
reverse('qgis_server:layer-request', kwargs=params), query_string)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
# OGC Server for the Geonode instance
# GetLegendGraphics is a shortcut when using the main OGC server.
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetLegendGraphics',
'FORMAT': 'image/png',
'LAYERS': uploaded.name,
}
response = self.client.get(
reverse('qgis_server:request'), query_string)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
# WMS GetCapabilities
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetCapabilities'
}
response = self.client.get(
reverse('qgis_server:request'), query_string)
self.assertEqual(response.status_code, 200, ensure_string(response.content))
self.assertEqual(
ensure_string(response.content), 'GetCapabilities is not supported yet.')
query_string['LAYERS'] = uploaded.name
response = self.client.get(
reverse('qgis_server:request'), query_string)
get_capabilities_content = ensure_string(response.content)
# Check xml content
self.assertEqual(response.status_code, 200, ensure_string(response.content))
root = dlxml.fromstring(ensure_string(response.content))
layer_xml = root.xpath(
'wms:Capability/wms:Layer/wms:Layer/wms:Name',
namespaces={'wms': 'http://www.opengis.net/wms'})
self.assertEqual(len(layer_xml), 1)
self.assertEqual(layer_xml[0].text, uploaded.name)
# GetLegendGraphic request returned must be valid
layer_xml = root.xpath(
'wms:Capability/wms:Layer/'
'wms:Layer/wms:Style/wms:LegendURL/wms:OnlineResource',
namespaces={
'xlink': 'http://www.w3.org/1999/xlink',
'wms': 'http://www.opengis.net/wms'
})
legend_url = layer_xml[0].attrib[
'{http://www.w3.org/1999/xlink}href']
response = self.client.get(legend_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
# Check get capabilities using helper returns the same thing
response = requests.get(wms_get_capabilities_url(
uploaded, internal=False))
self.assertEqual(response.status_code, 200)
self.assertEqual(get_capabilities_content, ensure_string(response.content))
# WMS GetMap
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetMap',
'FORMAT': 'image/png',
'LAYERS': uploaded.name,
'HEIGHT': 250,
'WIDTH': 250,
'SRS': 'EPSG:4326',
'BBOX': '-5.54025,96.9406,-5.2820,97.1250',
}
response = self.client.get(
reverse('qgis_server:request'), query_string)
self.assertEqual(response.status_code, 200, ensure_string(response.content))
self.assertEqual(
response.get('Content-Type'), 'image/png', ensure_string(response.content))
self.assertEqual(what('', h=ensure_string(response.content)), 'png')
# End of the test, we should remove every files related to the test.
uploaded.delete()
vector_layer.delete()
def create_models(type=None, integration=False):
users = []
obj_ids = []
with transaction.atomic():
map_data, user_data, people_data, dataset_data, document_data = create_fixtures()
registeredmembers_group, created = Group.objects.get_or_create(name='registered-members')
anonymous_group, created = Group.objects.get_or_create(name='anonymous')
cont_group, created = Group.objects.get_or_create(name='contributors')
perm = Permission.objects.get(codename='add_resourcebase')
cont_group.permissions.add(perm)
logger.debug("[SetUp] Get or create user admin")
u, created = get_user_model().objects.get_or_create(username='******')
u.set_password('admin')
u.is_superuser = True
u.first_name = 'admin'
u.save()
u.groups.add(anonymous_group)
users.append(u)
for ud, pd in zip(user_data, cycle(people_data)):
user_name, password, first_name, last_name = ud
logger.debug(f"[SetUp] Get or create user {user_name}")
u, created = get_user_model().objects.get_or_create(username=user_name)
u.set_password(password)
u.first_name = first_name
u.last_name = last_name
u.save()
u.groups.add(anonymous_group)
if not (u.is_superuser or u.is_staff or u.is_anonymous):
u.groups.add(cont_group)
users.append(u)
logger.debug(f"[SetUp] Add group {anonymous_group}")
get_user_model().objects.get(username='******').groups.add(anonymous_group)
from geonode.utils import DisableDjangoSignals
with DisableDjangoSignals(skip=integration):
if not type or ensure_string(type) == 'map':
for md, user in zip(map_data, cycle(users)):
title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), category = md
logger.debug(f"[SetUp] Add map {title}")
m = Map(
title=title,
abstract=abstract,
zoom=4,
projection='EPSG:4326',
center_x=42,
center_y=-73,
owner=user,
bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
srid='EPSG:4326',
category=category,
metadata_only=title == 'map metadata true'
)
m.save()
m.set_default_permissions()
m.clear_dirty_state()
m.set_processing_state(enumerations.STATE_PROCESSED)
obj_ids.append(m.id)
for kw in kws:
m.keywords.add(kw)
m.save()
if not type or ensure_string(type) == 'document':
for dd, user in zip(document_data, cycle(users)):
title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), category = dd
logger.debug(f"[SetUp] Add document {title}")
m = Document(
title=title,
abstract=abstract,
owner=user,
bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
srid='EPSG:4326',
files=dfile,
extension="gif",
metadata_only=title == 'doc metadata true'
)
m.save()
m.set_default_permissions()
m.clear_dirty_state()
m.set_processing_state(enumerations.STATE_PROCESSED)
obj_ids.append(m.id)
for kw in kws:
m.keywords.add(kw)
m.save()
if not type or ensure_string(type) == 'dataset':
for ld, owner, subtype in zip(dataset_data, cycle(users), cycle(('raster', 'vector'))):
title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ld
end = start + timedelta(days=365)
logger.debug(f"[SetUp] Add dataset {title}")
dataset = Dataset(
title=title,
abstract=abstract,
name=name,
alternate=alternate,
bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)),
srid='EPSG:4326',
uuid=str(uuid4()),
owner=owner,
temporal_extent_start=start,
temporal_extent_end=end,
date=start,
subtype=subtype,
category=category,
metadata_only=title == 'dataset metadata true'
)
dataset.save()
dataset.set_default_permissions()
dataset.clear_dirty_state()
dataset.set_processing_state(enumerations.STATE_PROCESSED)
obj_ids.append(dataset.id)
for kw in kws:
dataset.keywords.add(kw)
dataset.save()
return obj_ids
def test_add_delete_style(self):
"""Test add new style using qgis_server views."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
layer = file_upload(filename)
""":type: geonode.layers.models.Layer"""
self.client.login(username='******', password='******')
qml_path = self.data_path('test_grid.qml')
add_style_url = reverse(
'qgis_server:upload-qml',
kwargs={
'layername': layer.name})
with open(qml_path) as file_handle:
form_data = {
'name': 'new_style',
'title': 'New Style',
'qml': file_handle
}
response = self.client.post(
add_style_url,
data=form_data)
self.assertEqual(response.status_code, 201)
actual_list_style = style_list(layer, internal=False)
if actual_list_style:
expected_list_style = ['default', 'new_style']
self.assertEqual(
set(expected_list_style),
set([style.name for style in actual_list_style]))
# Test delete request
delete_style_url = reverse(
'qgis_server:remove-qml',
kwargs={
'layername': layer.name,
'style_name': 'default'})
response = self.client.delete(delete_style_url)
self.assertEqual(response.status_code, 200)
actual_list_style = style_list(layer, internal=False)
if actual_list_style:
expected_list_style = ['new_style']
self.assertEqual(
set(expected_list_style),
set([style.name for style in actual_list_style]))
# Check new default
default_style_url = reverse(
'qgis_server:default-qml',
kwargs={
'layername': layer.name})
response = self.client.get(default_style_url)
self.assertEqual(response.status_code, 200)
expected_default_style_retval = {
'name': 'new_style',
}
content = ensure_string(response.content)
if isinstance(content, bytes):
content = content.decode('UTF-8')
actual_default_style_retval = json.loads(content)
for key, value in expected_default_style_retval.items():
self.assertEqual(actual_default_style_retval[key], value)
layer.delete()
def test_map_thumbnail(self):
"""Creating map will create thumbnail."""
filename = os.path.join(
gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
layer1 = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
layer2 = file_upload(filename)
""":type: geonode.layers.models.Layer"""
# construct json request for new map
json_payload = InitialSetup.generate_initial_map(layer1, layer2)
self.client.login(username='******', password='******')
response = self.client.post(
reverse('new_map_json'),
json.dumps(json_payload),
content_type='application/json')
self.assertEqual(response.status_code, 200)
content = ensure_string(response.content)
if isinstance(content, bytes):
content = content.decode('UTF-8')
map_id = json.loads(content).get('id')
map = Map.objects.get(id=map_id)
# check that we have remote thumbnail
remote_thumbnail_link = map.link_set.filter(
name__icontains='remote thumbnail').first()
self.assertTrue(remote_thumbnail_link.url)
# thumbnail won't generate because remote thumbnail uses public
# address
remote_thumbnail_url = remote_thumbnail_link.url
# Replace url's basename, we want to access it using django client
parse_result = urlsplit(remote_thumbnail_url)
remote_thumbnail_url = urlunsplit(
('', '', parse_result.path, parse_result.query, ''))
response = self.client.get(remote_thumbnail_url)
thumbnail_path = thumb_path("map-thumb.png")
map.save_thumbnail(thumbnail_path, ensure_string(response.content))
# Check thumbnail created
self.assertTrue(storage.exists(thumbnail_path))
self.assertEqual(what(thumbnail_path), 'png')
# Check that now we have thumbnail
self.assertTrue(map.has_thumbnail())
missing_thumbnail_url = staticfiles.static(settings.MISSING_THUMBNAIL)
self.assertTrue(map.get_thumbnail_url() != missing_thumbnail_url)
thumbnail_links = map.link_set.filter(name__icontains='thumbnail')
self.assertTrue(len(thumbnail_links) > 0)
link_names = ['remote thumbnail', 'thumbnail']
for link in thumbnail_links:
self.assertIn(link.name.lower(), link_names)
# cleanup
map.delete()
layer1.delete()
layer2.delete()
def qml_style(request, layername, style_name=None):
"""Update/Retrieve QML style of a given QGIS Layer.
:param layername: The layer name in Geonode.
:type layername: basestring
:param style_name: The style name recognized by QGIS Server
:type style_name: str
"""
layer = get_object_or_404(Layer, name=layername)
if request.method == 'GET':
# Request QML from QGIS server
if not style_name:
# If no style name provided, then it is a List request
styles_obj = None
try:
styles_obj = style_list(layer, internal=False)
except Exception:
logger.info("Failed to fetch styles")
styles_dict = []
if styles_obj:
styles_dict = [model_to_dict(s) for s in styles_obj]
# If no style returned by GetCapabilities, this is a bug in QGIS
# Attempt to generate default style name
if not styles_dict:
style_url = style_get_url(layer, 'default')
response = requests.get(style_url)
if response.status_code == 200:
style_url = style_add_url(layer, 'default')
with open(layer.qgis_layer.qml_path, 'w') as f:
f.write(ensure_string(response.content))
response = requests.get(style_url)
if response.status_code == 200:
styles_obj = style_list(layer, internal=False)
styles_dict = [
model_to_dict(s) for s in styles_obj
]
response = HttpResponse(json.dumps(styles_dict),
content_type='application/json')
return response
# Return XML file of the style
style_url = style_get_url(layer, style_name, internal=False)
response = requests.get(style_url)
if response.status_code == 200:
response = HttpResponse(ensure_string(response.content),
content_type='text/xml')
response['Content-Disposition'] = 'attachment; filename=%s.qml' % (
style_name, )
else:
response = HttpResponse(ensure_string(response.content),
status=response.status_code)
return response
elif request.method == 'POST':
# For people who uses API request
if not request.user.has_perm('change_resourcebase',
layer.get_self_resource()):
return HttpResponse(
'User does not have permission to change QML style.',
status=403)
# Request about adding new QML style
form = QGISLayerStyleUploadForm(request.POST, request.FILES)
if not form.is_valid():
return TemplateResponse(request,
'qgis_server/forms/qml_style.html', {
'resource': layer,
'style_upload_form': form
},
status=400).render()
try:
uploaded_qml = request.FILES['qml']
# update qml in uploaded media folder
# check upload session, is qml file exists?
layerfile_set = layer.upload_session.layerfile_set
try:
qml_layer_file = layerfile_set.get(name='qml')
# if it is exists, we need to delete it, because it won't be
# managed by geonode
qml_layer_file.delete()
except LayerFile.DoesNotExist:
pass
# update qml in QGIS Layer folder
content = uploaded_qml.read()
qgis_layer = get_object_or_404(QGISServerLayer, layer=layer)
with open(qgis_layer.qml_path, mode='w') as f:
f.write(content)
# construct URL to post new QML
style_name = request.POST['name']
style_title = request.POST['title']
if not style_name:
# Assign default name
name_format = 'style_%Y%m%d%H%M%S'
current_time = datetime.datetime.utcnow()
style_name = current_time.strftime(name_format)
# Add new style
style_url = style_add_url(layer, style_name)
response = requests.get(style_url)
if not (response.status_code == 200
and ensure_string(response.content) == 'OK'):
try:
style_list(layer, internal=False)
except Exception:
logger.info("Failed to fetch styles")
return TemplateResponse(
request,
'qgis_server/forms/qml_style.html', {
'resource': layer,
'style_upload_form': QGISLayerStyleUploadForm(),
'alert': True,
'alert_message': ensure_string(response.content),
'alert_class': 'alert-danger'
},
status=response.status_code).render()
# We succeeded on adding new style
# Refresh style models
try:
style_list(layer, internal=False)
qgis_style = layer.qgis_layer.styles.get(name=style_name)
qgis_style.title = style_title
qgis_style.save()
alert_message = 'Successfully add style %s' % style_name
except Exception:
alert_message = 'Failed to fetch styles'
return TemplateResponse(request,
'qgis_server/forms/qml_style.html', {
'resource': layer,
'style_upload_form': form,
'alert': True,
'alert_class': 'alert-success',
'alert_message': alert_message
},
status=201).render()
except Exception as e:
logger.exception(e)
return HttpResponseServerError()
elif request.method == 'DELETE':
# Request to delete particular QML Style
if not style_name:
# Style name should exists
return HttpResponseBadRequest('Style name not provided.')
# Handle removing tile-style cache
try:
style = layer.qgis_layer.styles.get(name=style_name)
shutil.rmtree(style.style_tile_cache_path)
except Exception:
pass
style_url = style_remove_url(layer, style_name)
response = requests.get(style_url)
if not (response.status_code == 200
and ensure_string(response.content) == 'OK'):
alert_message = ensure_string(response.content)
if 'NAME is NOT an existing style.' in ensure_string(
response.content):
alert_message = '%s is not an existing style' % style_name
try:
style_list(layer, internal=False)
except Exception:
print("Failed to fetch styles")
return TemplateResponse(
request,
'qgis_server/forms/qml_style.html', {
'resource': layer,
'style_upload_form': QGISLayerStyleUploadForm(),
'alert': True,
'alert_message': alert_message,
'alert_class': 'alert-danger'
},
status=response.status_code).render()
# Successfully removed styles
# Handle when default style is deleted.
# Will be handled by style_list method
try:
style_list(layer, internal=False)
alert_message = 'Successfully deleted style %s' % style_name
except Exception:
alert_message = 'Failed to fetch styles'
return TemplateResponse(
request,
'qgis_server/forms/qml_style.html', {
'resource': layer,
'style_upload_form': QGISLayerStyleUploadForm(),
'alert': True,
'alert_message': alert_message,
'alert_class': 'alert-success'
},
status=200).render()
return HttpResponseBadRequest()
def default_qml_style(request, layername, style_name=None):
"""Set default style used by layer.
:param layername: The layer name in Geonode.
:type layername: basestring
:param style_name: The style name recognized by QGIS Server
:type style_name: str
"""
layer = get_object_or_404(Layer, name=layername)
if request.method == 'GET':
# Handle querying default style name request
default_style = layer.qgis_layer.default_style
retval = {
'name': default_style.name,
'title': default_style.title,
'style_url': default_style.style_url
}
return HttpResponse(json.dumps(retval),
content_type='application/json')
elif request.method == 'POST':
# For people who uses API request
if not request.user.has_perm('change_resourcebase',
layer.get_self_resource()):
return HttpResponse(
'User does not have permission to change QML style.',
status=403)
if not style_name:
return HttpResponseBadRequest()
style_url = style_set_default_url(layer, style_name)
response = requests.get(style_url)
if not (response.status_code == 200
and ensure_string(response.content) == 'OK'):
return HttpResponseServerError(
'Failed to change default Style.'
'Error: {0}'.format(ensure_string(response.content)))
# Succesfully change default style
# Synchronize models
style = layer.qgis_layer.styles.get(name=style_name)
qgis_layer = layer.qgis_layer
qgis_layer.default_style = style
qgis_layer.save()
alert_message = 'Successfully changed default style %s' % style_name
return TemplateResponse(
request,
'qgis_server/forms/qml_style.html', {
'resource': layer,
'style_upload_form': QGISLayerStyleUploadForm(),
'alert': True,
'alert_message': alert_message,
'alert_class': 'alert-success'
},
status=200).render()
def handle(self, url, name, type, method, console=sys.stdout, **options):
user = options.get('user')
owner = get_valid_user(user)
register_datasets = options.get('registerlayers')
username = options.get('username')
password = options.get('password')
perm_spec = options.get('permspec')
register_service = True
# First Check if this service already exists based on the URL
base_url = url
try:
service = Service.objects.get(base_url=base_url)
except Service.DoesNotExist:
service = None
if service is not None:
logger.info("This is an existing Service")
register_service = False
# Then Check that the name is Unique
try:
service = Service.objects.get(name=name)
except Service.DoesNotExist:
service = None
if service is not None:
print(
"This is an existing service using this name.\nPlease specify a different name."
)
if register_service:
if method == 'C':
response = _register_cascaded_service(type,
url,
name,
username,
password,
owner=owner,
verbosity=True)
elif method == 'I':
response = _register_indexed_service(type,
url,
name,
username,
password,
owner=owner,
verbosity=True)
elif method == 'H':
response = _register_harvested_service(url,
name,
username,
password,
owner=owner,
verbosity=True)
elif method == 'X':
print("Not Implemented (Yet)")
elif method == 'L':
print("Local Services not configurable via API")
else:
print("Invalid method")
json_response = json.loads(ensure_string(response.content))
if "id" in json_response:
print(f"Service created with id of {json_response['id']}")
service = Service.objects.get(id=json_response["id"])
else:
logger.error(
f"Something went wrong: {ensure_string(response.content)}")
return
print(service.id)
print(register_datasets)
if service and register_datasets:
layers = []
for layer in service.dataset_set.all():
layers.append(layer.alternate)
if service.method == 'C':
response = _register_cascaded_datasets(user, service, layers,
perm_spec)
elif service.method == 'I':
response = _register_indexed_datasets(user, service, layers,
perm_spec)
elif service.method == 'X':
print("Not Implemented (Yet)")
elif service.method == 'L':
print("Local Services not configurable via API")
else:
print("Invalid Service Type")
print(ensure_string(response.content))
def qgis_server_post_save_map(instance, sender, **kwargs):
"""Post Save Map Hook for QGIS Server
This hook will creates QGIS Project for a given map.
This hook also generates thumbnail link.
"""
logger.debug('QGIS Server Post Save Map custom')
map_id = instance.id
map_layers = MapLayer.objects.filter(map__id=map_id)
# Geonode map supports local layers and remote layers
# Remote layers were provided from other OGC services, so we don't
# deal with it at the moment.
local_layers = [_l for _l in map_layers if _l.local]
layers = []
for layer in local_layers:
try:
_l = Layer.objects.get(alternate=layer.name)
if not _l.qgis_layer:
raise QGISServerLayer.DoesNotExist
layers.append(_l)
except Layer.DoesNotExist:
msg = 'No Layer found for typename: {0}'.format(layer.name)
logger.debug(msg)
except QGISServerLayer.DoesNotExist:
msg = 'No QGIS Server Layer found for typename: {0}'.format(
layer.name)
logger.debug(msg)
if not layers:
# The signal is called too early, or the map has no layer yet.
return
base_url = settings.SITEURL
# download map
map_download_url = urljoin(
base_url, reverse('map_download', kwargs={'mapid': instance.id}))
logger.debug('map_download_url: %s' % map_download_url)
link_name = 'Download Data Layers'
Link.objects.update_or_create(resource=instance.resourcebase_ptr,
name=link_name,
defaults=dict(extension='html',
mime='text/html',
url=map_download_url,
link_type='data'))
# WMC map layer workspace
ogc_wmc_url = urljoin(base_url,
reverse('map_wmc', kwargs={'mapid': instance.id}))
logger.debug('wmc_download_url: %s' % ogc_wmc_url)
link_name = 'Download Web Map Context'
link_mime = 'application/xml'
Link.objects.update_or_create(resource=instance.resourcebase_ptr,
name=link_name,
defaults=dict(extension='wmc.xml',
mime=link_mime,
url=ogc_wmc_url,
link_type='data'))
# QLR map layer workspace
ogc_qlr_url = urljoin(
base_url, reverse('map_download_qlr', kwargs={'mapid': instance.id}))
logger.debug('qlr_map_download_url: %s' % ogc_qlr_url)
link_name = 'Download QLR Layer file'
link_mime = 'application/xml'
Link.objects.update_or_create(resource=instance.resourcebase_ptr,
name=link_name,
defaults=dict(extension='qlr',
mime=link_mime,
url=ogc_qlr_url,
link_type='data'))
# Set default bounding box based on all layers extents.
# bbox format [xmin, xmax, ymin, ymax]
bbox = instance.get_bbox_from_layers(instance.local_layers)
instance.set_bounds_from_bbox(bbox, instance.srid)
Map.objects.filter(id=map_id).update(bbox_polygon=Polygon.from_bbox(
(instance.bbox_x0, instance.bbox_x1, instance.bbox_y0,
instance.bbox_y1)),
srid=instance.srid,
zoom=instance.zoom,
center_x=instance.center_x,
center_y=instance.center_y)
# Check overwrite flag
overwrite = getattr(instance, 'overwrite', False)
# Create the QGIS Project
qgis_map, created = QGISServerMap.objects.get_or_create(map=instance)
# if it is newly created qgis_map, we should overwrite if existing files
# exists.
overwrite = created or overwrite
response = create_qgis_project(
layer=layers,
qgis_project_path=qgis_map.qgis_project_path,
overwrite=overwrite,
internal=True)
logger.debug('Create project url: {url}'.format(url=response.url))
logger.debug('Creating the QGIS Project : %s -> %s' %
(qgis_map.qgis_project_path, ensure_string(response.content)))
# Generate map thumbnail
create_qgis_server_thumbnail.delay(get_model_path(instance),
instance.id,
overwrite=True)
def qgis_server_request(request):
"""View to forward OGC request to QGIS Server."""
# Make a copy of the query string with capital letters for the key.
query = request.GET or request.POST
params = {param.upper(): value for param, value in query.items()}
# 900913 is deprecated
if params.get('SRS') == 'EPSG:900913':
params['SRS'] = 'EPSG:3857'
if params.get('CRS') == 'EPSG:900913':
params['CRS'] = 'EPSG:3857'
map_param = params.get('MAP')
# As we have one QGIS project per layer, we don't support GetCapabilities
# for now without any layer. We know, it's not OGC compliant.
if params.get('REQUEST') == 'GetCapabilities':
if (not map_param
and not (params.get('LAYERS') or params.get('TYPENAME'))):
return HttpResponse('GetCapabilities is not supported yet.')
# As we have one project per layer, we add the MAP path if the request is
# specific for one layer.
if not map_param and (params.get('LAYERS') or params.get('TYPENAME')):
# LAYERS is for WMS, TYPENAME for WFS
layer_name = params.get('LAYERS') or params.get('TYPENAME')
if len(layer_name.split(',')) > 1:
return HttpResponse('We do not support many layers in the request')
layer = get_object_or_404(Layer, name=layer_name)
qgis_layer = get_object_or_404(QGISServerLayer, layer=layer)
params['MAP'] = qgis_layer.qgis_project_path
# We have some shortcuts here instead of asking QGIS-Server.
if params.get('SERVICE') == 'WMS':
if params.get('REQUEST') == 'GetLegendGraphic':
layer_name = params.get('LAYER')
if not layer_name:
raise Http404('LAYER is not found for a GetLegendGraphic')
layer = get_object_or_404(Layer, name=layer_name)
return legend(request, layername=layer.name)
# Validation for STYLEMANAGER service
if params.get('SERVICE') == 'STYLEMANAGER':
project_param = params.get('PROJECT')
layer_name = params.get('LAYER')
if not project_param and layer_name:
layer = get_object_or_404(Layer, name=layer_name)
qgis_layer = get_object_or_404(QGISServerLayer, layer=layer)
params['PROJECT'] = qgis_layer.qgis_project_path
# if not shortcut, we forward any request to internal QGIS Server
qgis_server_url = qgis_server_endpoint(internal=True)
response = requests.get(qgis_server_url, params)
content = ensure_string(response.content)
# if it is GetCapabilities request, we need to replace all reference to
# our proxy
if params.get('REQUEST') == 'GetCapabilities':
qgis_server_base_url = qgis_server_endpoint(internal=True)
pattern = '{endpoint}'.format(endpoint=qgis_server_base_url)
content = re.sub(pattern, qgis_server_endpoint(internal=False),
content)
return HttpResponse(content,
content_type=response.headers.get('content-type'))
def create_models(type=None, integration=False):
from django.contrib.auth.models import Group
map_data, user_data, people_data, layer_data, document_data = create_fixtures(
)
anonymous_group, created = Group.objects.get_or_create(name='anonymous')
u, _ = get_user_model().objects.get_or_create(username='******',
is_superuser=True,
first_name='admin')
u.set_password('admin')
u.save()
users = []
for ud, pd in zip(user_data, cycle(people_data)):
user_name, password, first_name, last_name = ud
u, created = get_user_model().objects.get_or_create(username=user_name)
if created:
u.set_password(password)
u.first_name = first_name
u.last_name = last_name
u.save()
u.groups.add(anonymous_group)
users.append(u)
get_user_model().objects.get(
username='******').groups.add(anonymous_group)
obj_ids = []
from geonode.utils import DisableDjangoSignals
with DisableDjangoSignals(skip=integration):
if not type or ensure_string(type) == 'map':
for md, user in zip(map_data, cycle(users)):
title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0,
bbox_y1), category = md
m = Map(
title=title,
abstract=abstract,
zoom=4,
projection='EPSG:4326',
center_x=42,
center_y=-73,
owner=user,
bbox_x0=bbox_x0,
bbox_x1=bbox_x1,
bbox_y0=bbox_y0,
bbox_y1=bbox_y1,
srid='EPSG:4326',
category=category,
)
m.save()
obj_ids.append(m.id)
for kw in kws:
m.keywords.add(kw)
m.save()
if not type or ensure_string(type) == 'document':
for dd, user in zip(document_data, cycle(users)):
title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0,
bbox_y1), category = dd
m = Document(title=title,
abstract=abstract,
owner=user,
bbox_x0=bbox_x0,
bbox_x1=bbox_x1,
bbox_y0=bbox_y0,
bbox_y1=bbox_y1,
srid='EPSG:4326',
category=category,
doc_file=f)
m.save()
obj_ids.append(m.id)
for kw in kws:
m.keywords.add(kw)
m.save()
if not type or ensure_string(type) == 'layer':
for ld, owner, storeType in zip(
layer_data, cycle(users),
cycle(('coverageStore', 'dataStore'))):
title, abstract, name, alternate, (
bbox_x0, bbox_x1, bbox_y0,
bbox_y1), start, kws, category = ld
end = start + timedelta(days=365)
layer = Layer(title=title,
abstract=abstract,
name=name,
alternate=alternate,
bbox_x0=bbox_x0,
bbox_x1=bbox_x1,
bbox_y0=bbox_y0,
bbox_y1=bbox_y1,
srid='EPSG:4326',
uuid=str(uuid4()),
owner=owner,
temporal_extent_start=start,
temporal_extent_end=end,
date=start,
storeType=storeType,
category=category)
layer.save()
obj_ids.append(layer.id)
for kw in kws:
layer.keywords.add(kw)
layer.save()
return obj_ids
def update_from_viewer(self, conf, context=None):
"""
Update this Map's details by parsing a JSON object as produced by
a GXP Viewer.
This method automatically persists to the database!
"""
template_name = hookset.update_from_viewer(conf, context=context)
if not isinstance(context, dict):
try:
context = json.loads(ensure_string(context))
except Exception:
pass
conf = context.get("config", {})
if not isinstance(conf, dict) or isinstance(conf, bytes):
try:
conf = json.loads(ensure_string(conf))
except Exception:
conf = {}
about = conf.get("about", {})
self.title = conf.get("title", about.get("title", ""))
self.abstract = conf.get("abstract", about.get("abstract", ""))
_map = conf.get("map", {})
center = _map.get("center", settings.DEFAULT_MAP_CENTER)
self.zoom = _map.get("zoom", settings.DEFAULT_MAP_ZOOM)
if isinstance(center, dict):
self.center_x = center.get('x')
self.center_y = center.get('y')
else:
self.center_x, self.center_y = center
projection = _map.get("projection", settings.DEFAULT_MAP_CRS)
bbox = _map.get("bbox", None)
if bbox:
self.set_bounds_from_bbox(bbox, projection)
else:
self.set_bounds_from_center_and_zoom(
self.center_x,
self.center_y,
self.zoom)
if self.projection is None or self.projection == '':
self.projection = projection
if self.uuid is None or self.uuid == '':
self.uuid = str(uuid.uuid1())
def source_for(layer):
try:
return conf["sources"][layer["source"]]
except Exception:
if 'url' in layer:
return {'url': layer['url']}
else:
return {}
layers = [lyr for lyr in _map.get("layers", [])]
layer_names = set(lyr.alternate for lyr in self.local_layers)
self.layer_set.all().delete()
self.keywords.add(*_map.get('keywords', []))
for ordering, layer in enumerate(layers):
self.layer_set.add(
layer_from_viewer_config(
self.id, MapLayer, layer, source_for(layer), ordering
))
self.save(notify=True)
if layer_names != set(lyr.alternate for lyr in self.local_layers):
map_changed_signal.send_robust(sender=self, what_changed='layers')
return template_name