def test_layer_delete_from_geoserver(self):
"""Verify that layer is correctly deleted from GeoServer
"""
# Layer.delete() calls the pre_delete hook which uses cascading_delete()
# Should we explicitly test that the styles and store are
# deleted as well as the resource itself?
# There is already an explicit test for cascading delete
gs_cat = gs_catalog
# Test Uploading then Deleting a Shapefile from GeoServer
shp_file = os.path.join(
gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
shp_layer = file_upload(shp_file, overwrite=True)
shp_store = gs_cat.get_store(shp_layer.name)
shp_layer.delete()
self.assertRaises(
FailedRequestError,
lambda: gs_cat.get_resource(
shp_layer.name,
store=shp_store))
# Test Uploading then Deleting a TIFF file from GeoServer
tif_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
tif_layer = file_upload(tif_file)
tif_store = gs_cat.get_store(tif_layer.name)
tif_layer.delete()
self.assertRaises(
FailedRequestError,
lambda: gs_cat.get_resource(
shp_layer.name,
store=tif_store))
def test_unpublished(self):
"""Test permissions on an unpublished layer
"""
thefile = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_poi.shp")
layer = file_upload(thefile, overwrite=True)
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
# request getCapabilities: layer must be there as it is published and
# advertised: we need to check if in response there is
# <Name>geonode:san_andres_y_providencia_water</Name>
url = "http://localhost:8080/geoserver/ows?" "service=wms&version=1.3.0&request=GetCapabilities"
str_to_check = "<Name>geonode:san_andres_y_providencia_poi</Name>"
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertTrue(any(str_to_check in s for s in response.readlines()))
# by default the uploaded layer is
self.assertTrue(layer.is_published, True)
# Clean up and completely delete the layer
layer.delete()
# with settings disabled
with self.settings(RESOURCE_PUBLISHING=True):
thefile = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_administrative.shp")
layer = file_upload(thefile, overwrite=True)
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
str_to_check = "<Name>san_andres_y_providencia_administrative</Name>"
# by default the uploaded layer must be unpublished
self.assertEqual(layer.is_published, False)
# check the layer is not in GetCapabilities
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertFalse(any(str_to_check in s for s in response.readlines()))
# now test with published layer
resource = layer.get_self_resource()
resource.is_published = True
resource.save()
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertTrue(any(str_to_check in s for s in response.readlines()))
# Clean up and completely delete the layer
layer.delete()
def test_extension_not_implemented(self):
"""Verify a GeoNodeException is returned for not compatible extensions
"""
sampletxt = os.path.join(gisdata.VECTOR_DATA, "points_epsg2249_no_prj.dbf")
try:
file_upload(sampletxt)
except GeoNodeException, e:
pass
def save_shape(self, shape_in_memory=None, user=None):
if shape_in_memory is None:
shape_in_memory = self.shp
shape_in_memory.save(self.path_file)
self.__create_projection()
file = self.path_file + ".shp"
file_upload(file, user=user)
command_remove_shape = "rm -f " + self.path_file + ".*"
call(command_remove_shape, shell=True)
def test_bad_shapefile(self):
"""Verifying GeoNode complains about a shapefile without .prj
"""
thefile = os.path.join(gisdata.BAD_DATA, 'points_epsg2249_no_prj.shp')
try:
file_upload(thefile, overwrite=True)
except UploadError:
pass
except GeoNodeException:
pass
except Exception:
raise
def test_map_json(self):
# 2 layers to be added to the map
filename = os.path.join(
gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
layer1 = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
layer2 = file_upload(filename)
json_payload = InitialSetup.generate_initial_map(layer1, layer2)
# First, create a map with two layers
# Need to log in for saving a map
self.client.login(username='******', password='******')
result_new_map = self.client.post(
reverse('new_map_json'),
json.dumps(json_payload),
content_type='application/json')
# the new map is successfully saved
self.assertEqual(result_new_map.status_code, 200)
map_id = json.loads(result_new_map.content).get('id')
# try to remove one layer
layers = json_payload['map']['layers']
before_remove = len(layers)
after_remove = before_remove - 1
layer = layers[0]
layers.remove(layer)
# check if the layer is eliminated from the map
result_update_map = self.client.post(
reverse('map_json', kwargs={'mapid': map_id}),
data=json.dumps(json_payload),
content_type='application/json')
# successfully updated
self.assertEqual(result_update_map.status_code, 200)
# the number of layers on the map decrease by 1
self.assertEqual(
len(result_update_map.context_data['map'].layers),
after_remove)
# clean up
map = Map.objects.get(id=map_id)
map.delete()
layer1.delete()
layer2.delete()
def test_cascading_delete(self):
"""Verify that the helpers.cascading_delete() method is working properly
"""
gs_cat = gs_catalog
# Upload a Shapefile
shp_file = os.path.join(gisdata.VECTOR_DATA, 'san_andres_y_providencia_poi.shp')
shp_layer = file_upload(shp_file)
# Save the names of the Resource/Store/Styles
resource_name = shp_layer.name
ws = gs_cat.get_workspace(shp_layer.workspace)
store = gs_cat.get_store(shp_layer.store, ws)
store_name = store.name
layer = gs_cat.get_layer(resource_name)
styles = layer.styles + [layer.default_style]
# Delete the Layer using cascading_delete()
cascading_delete(gs_cat, shp_layer.typename)
# Verify that the styles were deleted
for style in styles:
s = gs_cat.get_style(style.name)
assert s == None
# Verify that the resource was deleted
self.assertRaises(FailedRequestError, lambda: gs_cat.get_resource(resource_name, store=store))
# Verify that the store was deleted
self.assertRaises(FailedRequestError, lambda: gs_cat.get_store(store_name))
# Clean up by deleting the layer from GeoNode's DB and GeoNetwork
shp_layer.delete()
def test_list_style(self):
"""Test querying list of styles from QGIS Server."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
layer = file_upload(filename)
""":type: geonode.layers.models.Layer"""
actual_list_style = style_list(layer, internal=False)
expected_list_style = ['default']
# There will be a default style
self.assertEqual(
set(expected_list_style),
set([style.name for style in actual_list_style]))
style_list_url = reverse(
'qgis_server:download-qml',
kwargs={
'layername': layer.name
})
response = self.client.get(style_list_url)
self.assertEqual(response.status_code, 200)
actual_list_style = json.loads(response.content)
# There will be a default style
self.assertEqual(
set(expected_list_style),
set([style['name'] for style in actual_list_style]))
layer.delete()
def test_map_thumbnail(self):
"""Test the map save method generates a thumbnail link
"""
# TODO: Would be nice to ensure the name is available before
# running the test...
norman = get_user_model().objects.get(username="******")
saved_layer = file_upload(
os.path.join(
gisdata.VECTOR_DATA,
"san_andres_y_providencia_poi.shp"),
name="san_andres_y_providencia_poi_by_norman",
user=norman,
overwrite=True,
)
try:
self.client.login(username='******', password='******')
saved_layer.set_default_permissions()
map_obj = Map(owner=norman, zoom=0,
center_x=0, center_y=0)
map_obj.create_from_layer_list(norman, [saved_layer], 'title', '')
thumbnail_url = map_obj.get_thumbnail_url()
self.assertNotEqual(thumbnail_url, staticfiles.static(settings.MISSING_THUMBNAIL))
finally:
# Cleanup
saved_layer.delete()
def test_empty_bbox(self):
"""Regression-test for failures caused by zero-width bounding boxes"""
thefile = os.path.join(gisdata.VECTOR_DATA, 'single_point.shp')
uploaded = file_upload(thefile, overwrite=True)
self.client.login(username='******', password='******')
resp = self.client.get(uploaded.get_absolute_url())
self.assertEquals(resp.status_code, 200)
def test_unpublished(self):
"""Test permissions on an unpublished layer
"""
thefile = os.path.join(
gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
layer = file_upload(thefile, overwrite=True)
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
# request getCapabilities: layer must be there as it is published and
# advertised: we need to check if in response there is
# <Name>geonode:san_andres_y_providencia_water</Name>
url = 'http://localhost:8080/geoserver/ows?' \
'service=wms&version=1.3.0&request=GetCapabilities'
str_to_check = '<Name>geonode:san_andres_y_providencia_poi</Name>'
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertTrue(any(str_to_check in s for s in response.readlines()))
# now test with unpublished layer
resource = layer.get_self_resource()
resource.is_published = False
resource.save()
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertFalse(any(str_to_check in s for s in response.readlines()))
# Clean up and completely delete the layer
layer.delete()
def test_getcapabilities_filters_by_site(self):
thefile1 = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_poi.shp")
thefile2 = os.path.join(gisdata.VECTOR_DATA, "san_andres_y_providencia_administrative.shp")
file_upload(thefile1, overwrite=True)
file_upload(thefile2, overwrite=True)
# remove one layer from the site resources
SiteResources.objects.get(site=self.slave_site).resources.remove(Layer.objects.all()[0])
self.assertEqual(SiteResources.objects.get(site=self.slave_site).resources.count(), 1)
self.client.login(username="******", password="******")
resp = self.client.get(self.getcaps_url)
xml = minidom.parseString(resp.content)
getcaps_layer = xml.getElementsByTagName("Layer")[0]
self.assertEqual(len(getcaps_layer.getElementsByTagName("Layer")), 1)
def test_set_attributes(self):
"""Test attributes syncronization
"""
# upload a shapefile
shp_file = os.path.join(
gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
layer = file_upload(shp_file)
# set attributes for resource
for attribute in layer.attribute_set.all():
attribute.attribute_label = '%s_label' % attribute.attribute
attribute.description = '%s_description' % attribute.attribute
attribute.save()
# sync the attributes with GeoServer
set_attributes(layer)
# tests if everything is synced properly
for attribute in layer.attribute_set.all():
self.assertEquals(
attribute.attribute_label,
'%s_label' % attribute.attribute
)
self.assertEquals(
attribute.description,
'%s_description' % attribute.attribute
)
def layer_replace(request, layername, template='layers/layer_replace.html'):
layer = _resolve_layer(request, layername, 'base.change_resourcebase',_PERMISSION_MSG_MODIFY)
if request.method == 'GET':
return render_to_response(template,
RequestContext(request, {'layer': layer,
'is_featuretype': layer.is_vector()}))
elif request.method == 'POST':
form = LayerUploadForm(request.POST, request.FILES)
tempdir = None
out = {}
if form.is_valid():
try:
tempdir, base_file = form.write_files()
saved_layer = file_upload(base_file, name=layer.name,
user=request.user, overwrite=True)
except Exception, e:
out['success'] = False
out['errors'] = str(e)
else:
out['success'] = True
out['url'] = reverse('layer_detail', args=[saved_layer.service_typename])
finally:
def layer_replace(request, layername, template='layers/layer_replace.html'):
layer = _resolve_layer(
request,
layername,
'base.change_resourcebase',
_PERMISSION_MSG_MODIFY)
if request.method == 'GET':
ctx = {
'charsets': CHARSETS,
'layer': layer,
'is_featuretype': layer.is_vector(),
'is_layer': True,
}
return render_to_response(template,
RequestContext(request, ctx))
elif request.method == 'POST':
form = LayerUploadForm(request.POST, request.FILES)
tempdir = None
out = {}
if form.is_valid():
try:
tempdir, base_file = form.write_files()
saved_layer = file_upload(
base_file,
name=layer.name,
user=request.user,
overwrite=True,
charset=form.cleaned_data["charset"],
)
except Exception as e:
out['success'] = False
out['errors'] = str(e)
else:
out['success'] = True
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
errormsgs = []
for e in form.errors.values():
errormsgs.append([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 500
return HttpResponse(
json.dumps(out),
mimetype='application/json',
status=status_code)
def test_layer_delete_from_geoserver(self):
"""Verify that layer is correctly deleted from GeoServer
"""
# Layer.delete() calls the pre_delete hook which uses cascading_delete()
# Should we explicitly test that the styles and store are
# deleted as well as the resource itself?
# There is already an explicit test for cascading delete
gs_cat = gs_catalog
# Test Uploading then Deleting a Shapefile from GeoServer
shp_file = os.path.join(
gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
shp_layer = file_upload(shp_file, overwrite=True)
# we need some time to have the service up and running
time.sleep(20)
ws = gs_cat.get_workspace(shp_layer.workspace)
shp_store = gs_cat.get_store(shp_layer.store, ws)
shp_store_name = shp_store.name
shp_layer.delete()
# Verify that it no longer exists in GeoServer
self.assertRaises(
FailedRequestError,
lambda: gs_cat.get_resource(
shp_layer.name,
store=shp_store))
# Verify that the store was deleted
ds = gs_cat.get_store(shp_store_name)
self.assertIsNone(ds)
# Test Uploading then Deleting a TIFF file from GeoServer
tif_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
tif_layer = file_upload(tif_file)
ws = gs_cat.get_workspace(tif_layer.workspace)
tif_store = gs_cat.get_store(tif_layer.store, ws)
tif_layer.delete()
self.assertRaises(
FailedRequestError,
lambda: gs_cat.get_resource(
shp_layer.name,
store=tif_store))
def layer_replace(request, layername, template='layers/layer_replace.html'):
layer = _resolve_layer(
request,
layername,
'base.change_resourcebase_permissions',
_PERMISSION_MSG_MODIFY)
try:
if "permission_denied" in layer:
return HttpResponse('You are not allowed to replace this layer',
mimetype="text/plain",
status=401)
except:
pass
if request.method == 'GET':
return render_to_response(
template, RequestContext(
request, {
'layer': layer, 'is_featuretype': layer.is_vector()}))
elif request.method == 'POST':
form = LayerUploadForm(request.POST, request.FILES)
tempdir = None
out = {}
if form.is_valid():
try:
tempdir, base_file = form.write_files()
saved_layer = file_upload(base_file, name=layer.name,
user=request.user, overwrite=True)
except Exception as e:
out['success'] = False
out['errors'] = str(e)
else:
out['success'] = True
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
errormsgs = []
for e in form.errors.values():
errormsgs.append([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 500
return HttpResponse(
json.dumps(out),
mimetype='application/json',
status=status_code)
def test_download_map_qlr(self):
"""Test download QLR file for a map"""
# 2 layers to be added to the map
filename = os.path.join(
gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
layer1 = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
layer2 = file_upload(filename)
# construct json request for new map
json_payload = InitialSetup.generate_initial_map(layer1, layer2)
self.client.login(username='******', password='******')
response = self.client.post(
reverse('new_map_json'),
json.dumps(json_payload),
content_type='application/json')
# map is successfull saved
self.assertEqual(response.status_code, 200)
map_id = json.loads(response.content).get('id')
map = Map.objects.get(id=map_id)
# check that QLR is added to the links
links = map.link_set.download()
map_qlr_link = links.get(name='Download QLR Layer file')
self.assertIn('qlr', map_qlr_link.url)
# QLR
response = self.client.get(
reverse('map_download_qlr', kwargs={'mapid': map_id}))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get('Content-Type'),
'application/x-qgis-layer-definition')
# cleanup
map.delete()
layer1.delete()
layer2.delete()
def test_tiff(self):
"""Uploading a good .tiff
"""
thefile = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
uploaded = file_upload(thefile, overwrite=True)
check_layer(uploaded)
# Clean up and completely delete the layer
uploaded.delete()
def setUp(self):
super(LayerMetadataDetailTest, self).setUp()
from geonode.layers.utils import file_upload
self.layer = file_upload(
os.path.join(TESTDIR, 'test_point.shp'),
name='testlayer'
)
self.url = '/layers/geonode:testlayer/metadata_detail'
def test_raster_upload(self):
"""Test that the wcs links are correctly created for a raster"""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
wcs_link = False
for link in uploaded.link_set.all():
if link.mime == 'GeoTIFF':
wcs_link = True
self.assertTrue(wcs_link)
def setUp(self):
super(NewMapConfigTest, self).setUp()
from geonode.layers.utils import file_upload
self.layer = file_upload(
os.path.join(TESTDIR, 'test_point.shp'),
name='testlayer'
)
self.url = '/layers/geonode:testlayer'
self.service_name = 'data-test'
def test_bad_shapefile(self):
"""Verifying GeoNode complains about a shapefile without .prj
"""
thefile = os.path.join(gisdata.BAD_DATA, "points_epsg2249_no_prj.shp")
try:
uploaded = file_upload(thefile, overwrite=True)
except GeoNodeException, e:
pass
def test_zipped_files(self):
"""Test that the zipped files is created for raster."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
zip_link = False
for link in uploaded.link_set.all():
if link.mime == 'ZIP':
zip_link = True
self.assertTrue(zip_link)
def test_shapefile(self):
"""Test Uploading a good shapefile
"""
thefile = os.path.join(gisdata.VECTOR_DATA, 'san_andres_y_providencia_poi.shp')
uploaded = file_upload(thefile, overwrite=True)
check_layer(uploaded)
# Clean up and completely delete the layer
uploaded.delete()
def test_keywords_upload(self):
"""Check that keywords can be passed to file_upload
"""
thefile = os.path.join(gisdata.VECTOR_DATA, 'san_andres_y_providencia_poi.shp')
uploaded = file_upload(thefile, keywords=['foo', 'bar'], overwrite=True)
keywords = uploaded.keyword_list()
msg='No keywords found in layer %s' % uploaded.name
assert len(keywords)>0, msg
assert 'foo' in uploaded.keyword_list(), 'Could not find "foo" in %s' % keywords
assert 'bar' in uploaded.keyword_list(), 'Could not find "bar" in %s' % keywords
def setUp(self):
super(LayerPublishTest, self).setUp()
from geonode.layers.utils import file_upload
self.layer = file_upload(
os.path.join(TESTDIR, 'test_point.shp'),
name='testlayer'
)
self.url = '/layers/geonode:testlayer/publish'
# We expect to be redirected
self.expected_status = 302
def create_layer(number):
""" Creates a new layer """
file_list = glob.glob('%s*.shp' % shp_path)
random_index = randint(0, len(file_list) -1)
file_uri = file_list[random_index]
layer = file_upload(file_uri)
# keywords
assign_keywords(layer)
# other stuff
resource = layer.get_self_resource()
set_resource(resource)
def layer_replace(request, layername, template="layers/layer_replace.html"):
layer = _resolve_layer(request, layername, "base.change_resourcebase", _PERMISSION_MSG_MODIFY)
if request.method == "GET":
ctx = {"charsets": CHARSETS, "layer": layer, "is_featuretype": layer.is_vector(), "is_layer": True}
return render_to_response(template, RequestContext(request, ctx))
elif request.method == "POST":
form = LayerUploadForm(request.POST, request.FILES)
tempdir = None
out = {}
if form.is_valid():
try:
tempdir, base_file = form.write_files()
if layer.is_vector() and is_raster(base_file):
out["success"] = False
out["errors"] = _("You are attempting to replace a vector layer with a raster.")
elif (not layer.is_vector()) and is_vector(base_file):
out["success"] = False
out["errors"] = _("You are attempting to replace a raster layer with a vector.")
else:
# delete geoserver's store before upload
cat = gs_catalog
cascading_delete(cat, layer.typename)
saved_layer = file_upload(
base_file,
name=layer.name,
user=request.user,
overwrite=True,
charset=form.cleaned_data["charset"],
)
out["success"] = True
out["url"] = reverse("layer_detail", args=[saved_layer.service_typename])
except Exception as e:
out["success"] = False
out["errors"] = str(e)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
errormsgs = []
for e in form.errors.values():
errormsgs.append([escape(v) for v in e])
out["errors"] = form.errors
out["errormsgs"] = errormsgs
if out["success"]:
status_code = 200
else:
status_code = 400
return HttpResponse(json.dumps(out), mimetype="application/json", status=status_code)
def test_repeated_upload(self):
"""Upload the same file more than once
"""
thefile = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
uploaded1 = file_upload(thefile, overwrite=True)
check_layer(uploaded1)
uploaded2 = file_upload(thefile, overwrite=True)
check_layer(uploaded2)
uploaded3 = file_upload(thefile, overwrite=False)
check_layer(uploaded3)
msg = ('Expected %s but got %s' % (uploaded1.name, uploaded2.name))
assert uploaded1.name == uploaded2.name, msg
msg = ('Expected a different name when uploading %s using '
'overwrite=False but got %s' % (thefile, uploaded3.name))
assert uploaded1.name != uploaded3.name, msg
# Clean up and completely delete the layers
# uploaded1 is overwritten by uploaded2 ... no need to delete it
uploaded2.delete()
uploaded3.delete()
def test_ogc_specific_layer(self):
"""Test we can use QGIS Server API for a layer.
For now, we are just checking we can call these views without any
exceptions. We should improve this test by checking the result.
"""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
vector_layer = file_upload(filename)
params = {'layername': uploaded.name}
# Zip
response = self.client.get(
reverse('qgis_server:download-zip', kwargs=params))
self.assertEqual(response.status_code, 200)
try:
f = StringIO.StringIO(response.content)
zipped_file = zipfile.ZipFile(f, 'r')
for one_file in zipped_file.namelist():
# We shoudn't get any QGIS project
self.assertFalse(one_file.endswith('.qgs'))
self.assertIsNone(zipped_file.testzip())
finally:
zipped_file.close()
f.close()
# Legend
response = self.client.get(
reverse('qgis_server:legend', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=response.content), 'png')
# Tile
coordinates = {'z': '11', 'x': '1576', 'y': '1054'}
coordinates.update(params)
response = self.client.get(
reverse('qgis_server:tile', kwargs=coordinates))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=response.content), 'png')
# Tile 404
response = self.client.get(
reverse('qgis_server:tile', kwargs=params))
self.assertEqual(response.status_code, 404)
self.assertEqual(
response.get('Content-Type'), 'text/html; charset=utf-8')
# Geotiff
response = self.client.get(
reverse('qgis_server:geotiff', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/tiff')
self.assertEqual(what('', h=response.content), 'tiff')
# Layer is already on the database
# checking the Link
links = uploaded.link_set.download().filter(
name__in=settings.DOWNLOAD_FORMATS_RASTER)
# checks signals.py for the hardcoded names in QLR and QGS
qlr_link = links.get(name='QGIS layer file (.qlr)')
self.assertIn("download-qlr", qlr_link.url)
qgs_link = links.get(name='QGIS project file (.qgs)')
self.assertIn("download-qgs", qgs_link.url)
# QLR
response = self.client.get(
reverse('qgis_server:download-qlr', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get('Content-Type'),
'application/x-qgis-layer-definition')
# check file name's extension
file_name = response.get('Content-Disposition').split('filename=')
file_ext = file_name[1].split('.')
self.assertEqual(file_ext[1], "qlr")
# QGS
response = self.client.get(
reverse('qgis_server:download-qgs', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get('Content-Type'),
'application/x-qgis-project')
# check file name's extension
file_name = response.get('Content-Disposition').split('filename=')
file_ext = file_name[1].split('.')
self.assertEqual(file_ext[1], "qgs")
response = self.client.get(
reverse('qgis_server:geotiff', kwargs={
'layername': vector_layer.name
}))
self.assertEqual(response.status_code, 404)
# QML Styles
# Request list of styles
response = self.client.get(
reverse('qgis_server:download-qml', kwargs=params))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'application/json')
# Should return a default style list
actual_result = json.loads(response.content)
actual_result = [s['name'] for s in actual_result]
expected_result = ['default']
self.assertEqual(set(expected_result), set(actual_result))
# Get single styles
response = self.client.get(
reverse('qgis_server:download-qml', kwargs={
'layername': params['layername'],
'style_name': 'default'
}))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'text/xml')
# Set thumbnail from viewed bbox
response = self.client.get(
reverse('qgis_server:set-thumbnail', kwargs=params))
self.assertEqual(response.status_code, 400)
data = {
'bbox': '-5.54025,96.9406,-5.2820,97.1250'
}
response = self.client.post(
reverse('qgis_server:set-thumbnail', kwargs=params),
data=data)
# User dont have permission
self.assertEqual(response.status_code, 403)
# Should log in
self.client.login(username='******', password='******')
response = self.client.post(
reverse('qgis_server:set-thumbnail', kwargs=params),
data=data)
self.assertEqual(response.status_code, 200)
retval = json.loads(response.content)
expected_retval = {
'success': True
}
self.assertEqual(retval, expected_retval)
# OGC Server specific for THE layer
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetLegendGraphics',
'FORMAT': 'image/png',
'LAYERS': uploaded.name,
}
response = self.client.get(
reverse('qgis_server:layer-request', kwargs=params), query_string)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=response.content), 'png')
# OGC Server for the Geonode instance
# GetLegendGraphics is a shortcut when using the main OGC server.
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetLegendGraphics',
'FORMAT': 'image/png',
'LAYERS': uploaded.name,
}
response = self.client.get(
reverse('qgis_server:request'), query_string)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=response.content), 'png')
# WMS GetCapabilities
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetCapabilities'
}
response = self.client.get(
reverse('qgis_server:request'), query_string)
self.assertEqual(response.status_code, 200, response.content)
self.assertEqual(
response.content, 'GetCapabilities is not supported yet.')
query_string['LAYERS'] = uploaded.name
response = self.client.get(
reverse('qgis_server:request'), query_string)
get_capabilities_content = response.content
# Check xml content
self.assertEqual(response.status_code, 200, response.content)
root = etree.fromstring(response.content)
layer_xml = root.xpath(
'wms:Capability/wms:Layer/wms:Layer/wms:Name',
namespaces={'wms': 'http://www.opengis.net/wms'})
self.assertEqual(len(layer_xml), 1)
self.assertEqual(layer_xml[0].text, uploaded.name)
# GetLegendGraphic request returned must be valid
layer_xml = root.xpath(
'wms:Capability/wms:Layer/'
'wms:Layer/wms:Style/wms:LegendURL/wms:OnlineResource',
namespaces={
'xlink': 'http://www.w3.org/1999/xlink',
'wms': 'http://www.opengis.net/wms'
})
legend_url = layer_xml[0].attrib[
'{http://www.w3.org/1999/xlink}href']
response = self.client.get(legend_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get('Content-Type'), 'image/png')
self.assertEqual(what('', h=response.content), 'png')
# Check get capabilities using helper returns the same thing
response = requests.get(wms_get_capabilities_url(
uploaded, internal=False))
self.assertEqual(response.status_code, 200)
self.assertEqual(get_capabilities_content, response.content)
# WMS GetMap
query_string = {
'SERVICE': 'WMS',
'VERSION': '1.3.0',
'REQUEST': 'GetMap',
'FORMAT': 'image/png',
'LAYERS': uploaded.name,
'HEIGHT': 250,
'WIDTH': 250,
'SRS': 'EPSG:4326',
'BBOX': '-5.54025,96.9406,-5.2820,97.1250',
}
response = self.client.get(
reverse('qgis_server:request'), query_string)
self.assertEqual(response.status_code, 200, response.content)
self.assertEqual(
response.get('Content-Type'), 'image/png', response.content)
self.assertEqual(what('', h=response.content), 'png')
# End of the test, we should remove every files related to the test.
uploaded.delete()
vector_layer.delete()
def test_layer_replace(self):
"""Test layer replace functionality
"""
vector_file = os.path.join(
gisdata.VECTOR_DATA, 'san_andres_y_providencia_administrative.shp')
vector_layer = file_upload(vector_file, overwrite=True)
raster_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
raster_layer = file_upload(raster_file, overwrite=True)
# we need some time to have the service up and running
time.sleep(20)
new_vector_layer = None
try:
self.client.login(username='******', password='******')
# test the program can determine the original layer in raster type
raster_replace_url = reverse('layer_replace',
args=[raster_layer.service_typename])
response = self.client.get(raster_replace_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(response.context['is_featuretype'], False)
# test the program can determine the original layer in vector type
vector_replace_url = reverse('layer_replace',
args=[vector_layer.service_typename])
response = self.client.get(vector_replace_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(response.context['is_featuretype'], True)
# test replace a vector with a raster
response = self.client.post(vector_replace_url,
{'base_file': open(raster_file, 'rb')})
# TODO: This should really return a 400 series error with the json dict
self.assertEquals(response.status_code, 400)
response_dict = json.loads(response.content)
self.assertEquals(response_dict['success'], False)
# test replace a vector with a different vector
new_vector_file = os.path.join(
gisdata.VECTOR_DATA, 'san_andres_y_providencia_coastline.shp')
layer_path, __ = os.path.splitext(new_vector_file)
layer_base = open(layer_path + '.shp', 'rb')
layer_dbf = open(layer_path + '.dbf', 'rb')
layer_shx = open(layer_path + '.shx', 'rb')
layer_prj = open(layer_path + '.prj', 'rb')
response = self.client.post(
vector_replace_url, {
'base_file': layer_base,
'dbf_file': layer_dbf,
'shx_file': layer_shx,
'prj_file': layer_prj,
'charset': 'UTF-8'
})
response_dict = json.loads(response.content)
if not response_dict[
'success'] and 'unknown encoding' in response_dict[
'errors']:
print(response_dict['errors'])
pass
else:
self.assertEquals(response.status_code, 200)
self.assertEquals(response_dict['success'], True)
# Get a Layer object for the newly created layer.
new_vector_layer = Layer.objects.get(pk=vector_layer.pk)
# FIXME(Ariel): Check the typename does not change.
# Test the replaced layer is indeed different from the original layer
self.assertNotEqual(vector_layer.bbox_x0,
new_vector_layer.bbox_x0)
self.assertNotEqual(vector_layer.bbox_x1,
new_vector_layer.bbox_x1)
self.assertNotEqual(vector_layer.bbox_y0,
new_vector_layer.bbox_y0)
self.assertNotEqual(vector_layer.bbox_y1,
new_vector_layer.bbox_y1)
# test an invalid user without layer replace permission
self.client.logout()
self.client.login(username='******', password='******')
response = self.client.post(
vector_replace_url, {
'base_file': layer_base,
'dbf_file': layer_dbf,
'shx_file': layer_shx,
'prj_file': layer_prj
})
self.assertEquals(response.status_code, 401)
finally:
# Clean up and completely delete the layer
try:
if vector_layer:
vector_layer.delete()
if raster_layer:
raster_layer.delete()
if new_vector_layer:
new_vector_layer.delete()
except:
tb = traceback.format_exc()
print(tb)
def process_impact_result(self, impact_url, analysis_id):
"""Extract impact analysis after running it via InaSAFE-Headless celery
:param self: Task instance
:type self: celery.task.Task
:param impact_url: impact url returned from analysis
:type impact_url: str
:param analysis_id: analysis id of the object
:type analysis_id: int
:return: True if success
:rtype: bool
"""
# Track the current task_id
analysis = Analysis.objects.get(id=analysis_id)
analysis.task_id = self.request.id
analysis.save()
# decide if we are using direct access or not
impact_url = get_impact_path(impact_url)
# download impact zip
impact_path = download_file(impact_url)
dir_name = os.path.dirname(impact_path)
success = False
with ZipFile(impact_path) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if ext in ['.shp', '.tif']:
# process this in the for loop to make sure it works only
# when we found the layer
saved_layer = file_upload(os.path.join(dir_name, name),
overwrite=True)
saved_layer.set_default_permissions()
if analysis.user_title:
layer_name = analysis.user_title
else:
layer_name = analysis.get_default_impact_title()
saved_layer.title = layer_name
saved_layer.save()
current_impact = None
if analysis.impact_layer:
current_impact = analysis.impact_layer
analysis.impact_layer = saved_layer
# check map report and table
report_map_path = os.path.join(dir_name, '%s.pdf' % basename)
if os.path.exists(report_map_path):
analysis.assign_report_map(report_map_path)
report_table_path = os.path.join(dir_name,
'%s_table.pdf' % basename)
if os.path.exists(report_table_path):
analysis.assign_report_table(report_table_path)
analysis.task_id = process_impact_result.request.id
analysis.task_state = 'SUCCESS'
analysis.end_time = datetime.now().strftime('%Y-%m-%d %H:%M')
analysis.save()
if current_impact:
current_impact.delete()
success = True
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except BaseException:
pass
# cleanup
try:
os.remove(impact_path)
except BaseException:
pass
if not success:
LOGGER.info('No impact layer found in %s' % impact_url)
return success
def test_layer_upload_metadata(self):
"""Test uploading a layer with XML metadata"""
thelayer = os.path.join(
gisdata.PROJECT_ROOT,
'both',
'good',
'sangis.org',
'Airport',
'Air_Runways.shp')
self.assertTrue('%s.xml' % thelayer,
'Expected layer XML metadata to exist')
uploaded = file_upload(thelayer, overwrite=True)
self.assertEqual(
uploaded.title,
'Air_Runways',
'Expected specific title from uploaded layer XML metadata')
self.assertEqual(
uploaded.abstract,
'Airport Runways',
'Expected specific abstract from uploaded layer XML metadata')
self.assertEqual(
uploaded.purpose,
'To show the location of Public Airports '
'and Runways within San Diego County',
'Expected specific purpose from uploaded layer XML metadata')
self.assertEqual(uploaded.supplemental_information,
'No information provided',
'Expected specific supplemental information '
'from uploaded layer XML metadata')
self.assertEqual(len(uploaded.keyword_list(
)), 5, 'Expected specific number of keywords from uploaded layer XML metadata')
self.assertTrue(
'Landing Strips' in uploaded.keyword_list(),
'Expected specific keyword from uploaded layer XML metadata')
self.assertEqual(
uploaded.constraints_other,
'None',
'Expected specific constraint from uploaded layer XML metadata')
self.assertEqual(
uploaded.date,
datetime.datetime(
2010,
8,
3,
0,
0),
'Expected specific date from uploaded layer XML metadata')
# Clean up and completely delete the layer
uploaded.delete()
def data_request_view(request):
profile_request_obj = request.session.get('profile_request_obj', None)
if not profile_request_obj:
pprint("no profile request object found")
if not request.user.is_authenticated() and not profile_request_obj:
return redirect(reverse('datarequests:profile_request_form'))
request.session['data_request_session'] = True
form = DataRequestForm()
if request.method == 'POST' :
pprint("detected data request post")
post_data = request.POST.copy()
post_data['permissions'] = '{"users":{"dataRegistrationUploader": ["view_resourcebase"] }}'
data_classes = post_data.getlist('data_class_requested')
#data_classes = post_data.get('data_class_requested')
data_class_objs = []
pprint(data_classes)
pprint("len:"+str(len(data_classes)))
if len(data_classes) == 1:
post_data.setlist('data_class_requested',data_classes[0].replace('[','').replace(']','').replace('"','').split(','))
pprint(post_data.getlist('data_class_requested'))
details_form = DataRequestForm(post_data, request.FILES)
data_request_obj = None
errormsgs = []
out = {}
out['errors'] = {}
out['success'] = False
saved_layer = None
if not details_form.is_valid():
for e in details_form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['errors'] = dict(
(k, map(unicode, v))
for (k,v) in details_form.errors.iteritems())
pprint(out['errors'])
else:
tempdir = None
shapefile_form = DataRequestShapefileForm(post_data, request.FILES)
if u'base_file' in request.FILES:
if shapefile_form.is_valid():
title = shapefile_form.cleaned_data["layer_title"]
# Replace dots in filename - GeoServer REST API upload bug
# and avoid any other invalid characters.
# Use the title if possible, otherwise default to the filename
if title is not None and len(title) > 0:
name_base = title
else:
name_base, __ = os.path.splitext(
shapefile_form.cleaned_data["base_file"].name)
name = slugify(name_base.replace(".", "_"))
try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
# This should be followed up in upstream Django.
tempdir, base_file = shapefile_form.write_files()
registration_uploader, created = Profile.objects.get_or_create(username='******')
pprint("saving jurisdiction")
saved_layer = file_upload(
base_file,
name=name,
user=registration_uploader,
overwrite=False,
charset=shapefile_form.cleaned_data["charset"],
abstract=shapefile_form.cleaned_data["abstract"],
title=shapefile_form.cleaned_data["layer_title"],
)
except Exception as e:
exception_type, error, tb = sys.exc_info()
print traceback.format_exc()
out['success'] = False
out['errors'] = "An unexpected error was encountered. Check the files you have uploaded, clear your selected files, and try again."
# Assign the error message to the latest UploadSession of the data request uploader user.
latest_uploads = UploadSession.objects.filter(
user=registration_uploader
).order_by('-date')
if latest_uploads.count() > 0:
upload_session = latest_uploads[0]
upload_session.error = str(error)
upload_session.traceback = traceback.format_exc(tb)
upload_session.context = "Data requester's jurisdiction file upload"
upload_session.save()
out['traceback'] = upload_session.traceback
out['context'] = upload_session.context
out['upload_session'] = upload_session.id
else:
pprint("layer_upload is successful")
out['success'] = True
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
upload_session = saved_layer.upload_session
upload_session.processed = True
upload_session.save()
permissions = {
'users': {'dataRegistrationUploader': []},
'groups': {}
}
if request.user.is_authenticated():
permissions = {
'users': {request.user.username : ['view_resourcebase']},
'groups': {}
}
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
jurisdiction_style.delay(saved_layer)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
for e in shapefile_form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['success'] = False
out['errors'].update(dict(
(k, map(unicode, v))
for (k,v) in shapefile_form.errors.iteritems()
))
pprint(out['errors'])
out['errormsgs'] = out['errors']
#if out['success']:
if not out['errors']:
out['success'] = True
data_request_obj = details_form.save()
if request.user.is_authenticated() and not request.user.username == 'AnonymousUser':
request_letter = create_letter_document(details_form.clean()['letter_file'], profile = request.user)
data_request_obj.request_letter = request_letter
data_request_obj.save()
data_request_obj.profile = request.user
data_request_obj.save()
data_request_obj.set_status("pending")
else:
request_letter = create_letter_document(details_form.clean()['letter_file'], profile_request = profile_request_obj)
data_request_obj.request_letter = request_letter
data_request_obj.save()
data_request_obj.profile_request = profile_request_obj
data_request_obj.save()
profile_request_obj.data_request= data_request_obj
profile_request_obj.save()
data_request_obj.save()
if saved_layer:
data_request_obj.jurisdiction_shapefile = saved_layer
data_request_obj.save()
place_name_update.delay([data_request_obj])
compute_size_update.delay([data_request_obj])
tag_request_suc.delay([data_request_obj])
status_code = 200
pprint("request has been succesfully submitted")
if profile_request_obj and not request.user.is_authenticated():
# request_data.send_verification_email()
out['success_url'] = reverse('datarequests:email_verification_send')
out['redirect_to'] = reverse('datarequests:email_verification_send')
elif request.user.is_authenticated():
messages.info(request, "Your request has been submitted")
out['success_url'] = reverse('home')
out['redirect_to'] = reverse('home')
del request.session['data_request_session']
if 'profile_request_obj' in request.session:
del request.session['profile_request_obj']
else:
status_code = 400
pprint("sending this out")
return HttpResponse(
json.dumps(out),
mimetype='application/json',
status=status_code)
pprint("i'm here")
return render(
request,
'datarequests/registration/shapefile.html',
{
'charsets': CHARSETS,
'is_layer': True,
'form': form,
'support_email': settings.LIPAD_SUPPORT_MAIL,
})
def test_delete_orphan(self):
"""Test orphan deletions.
This test only started in travis to avoid accidentally deleting owners
data.
"""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
# Clean up first
call_command('delete_orphaned_qgis_server_layers')
# make request to generate tile cache
response = self.client.get(
reverse('qgis_server:tile', kwargs={
'z': '11',
'x': '1576',
'y': '1054',
'layername': uploaded.name
}))
self.assertEqual(response.status_code, 200)
# register file list
layer_path = settings.QGIS_SERVER_CONFIG['layer_directory']
tiles_path = settings.QGIS_SERVER_CONFIG['tiles_directory']
geonode_layer_path = os.path.join(settings.MEDIA_ROOT, 'layers')
qgis_layer_list = set(os.listdir(layer_path))
tile_cache_list = set(os.listdir(tiles_path))
geonode_layer_list = set(os.listdir(geonode_layer_path))
# run management command. should not change anything
call_command('delete_orphaned_qgis_server_layers')
actual_qgis_layer_list = set(os.listdir(layer_path))
actual_tile_cache_list = set(os.listdir(tiles_path))
actual_geonode_layer_list = set(os.listdir(geonode_layer_path))
self.assertEqual(qgis_layer_list, actual_qgis_layer_list)
self.assertEqual(tile_cache_list, actual_tile_cache_list)
self.assertEqual(geonode_layer_list, actual_geonode_layer_list)
# now create random file without reference
shutil.copy(
os.path.join(layer_path, 'test_grid.tif'),
os.path.join(layer_path, 'test_grid_copy.tif'))
shutil.copytree(
os.path.join(tiles_path, 'test_grid'),
os.path.join(tiles_path, 'test_grid_copy'))
shutil.copy(
os.path.join(geonode_layer_path, 'test_grid.tif'),
os.path.join(geonode_layer_path, 'test_grid_copy.tif'))
actual_qgis_layer_list = set(os.listdir(layer_path))
actual_tile_cache_list = set(os.listdir(tiles_path))
actual_geonode_layer_list = set(os.listdir(geonode_layer_path))
# run management command. This should clear the files. But preserve
# registered files (the one that is saved in database)
call_command('delete_orphaned_qgis_server_layers')
self.assertEqual(
{'test_grid_copy.tif'},
actual_qgis_layer_list - qgis_layer_list)
self.assertEqual(
{'test_grid_copy'},
actual_tile_cache_list - tile_cache_list)
self.assertEqual(
{'test_grid_copy.tif'},
actual_geonode_layer_list - geonode_layer_list)
# cleanup
uploaded.delete()
def test_capabilities(self):
"""Test capabilities
"""
# a category
category = TopicCategory.objects.all()[0]
# some users
norman = get_user_model().objects.get(username="******")
admin = get_user_model().objects.get(username="******")
# create 3 layers, 2 with norman as an owner an 2 with category as a category
layer1 = file_upload(
os.path.join(gisdata.VECTOR_DATA,
"san_andres_y_providencia_poi.shp"),
name='layer1',
user=norman,
category=category,
overwrite=True,
)
layer2 = file_upload(
os.path.join(gisdata.VECTOR_DATA, "single_point.shp"),
name='layer2',
user=norman,
overwrite=True,
)
layer3 = file_upload(
os.path.join(gisdata.VECTOR_DATA,
"san_andres_y_providencia_administrative.shp"),
name='layer3',
user=admin,
category=category,
overwrite=True,
)
try:
namespaces = {
'wms': 'http://www.opengis.net/wms',
'xlink': 'http://www.w3.org/1999/xlink',
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
}
# 0. test capabilities_layer
url = reverse('capabilities_layer', args=[layer1.id])
resp = self.client.get(url)
layercap = dlxml.fromstring(resp.content)
rootdoc = etree.ElementTree(layercap)
layernodes = rootdoc.findall('./[wms:Name]', namespaces)
layernode = layernodes[0]
self.assertEqual(1, len(layernodes))
self.assertEqual(
layernode.find('wms:Name', namespaces).text,
"geonode:{}".format(layer1.name))
# 1. test capabilities_user
url = reverse('capabilities_user', args=[norman.username])
resp = self.client.get(url)
layercap = dlxml.fromstring(resp.content)
rootdoc = etree.ElementTree(layercap)
layernodes = rootdoc.findall('./[wms:Name]', namespaces)
# norman has 2 layers
self.assertEqual(1, len(layernodes))
# the norman two layers are named layer1 and layer2
count = 0
for layernode in layernodes:
if layernode.find(
'wms:Name',
namespaces).text == '%s:%s' % ('geonode', layer1.name):
count += 1
elif layernode.find(
'wms:Name',
namespaces).text == '%s:%s' % ('geonode', layer2.name):
count += 1
self.assertEqual(1, count)
# 2. test capabilities_category
url = reverse('capabilities_category', args=[category.identifier])
resp = self.client.get(url)
layercap = dlxml.fromstring(resp.content)
rootdoc = etree.ElementTree(layercap)
layernodes = rootdoc.findall('./[wms:Name]', namespaces)
# category is in two layers
self.assertEqual(1, len(layernodes))
# the layers for category are named layer1 and layer3
count = 0
for layernode in layernodes:
if layernode.find(
'wms:Name',
namespaces).text == '%s:%s' % ('geonode', layer1.name):
count += 1
elif layernode.find(
'wms:Name',
namespaces).text == '%s:%s' % ('geonode', layer3.name):
count += 1
self.assertEqual(1, count)
# 3. test for a map
# TODO
finally:
# Clean up and completely delete the layers
layer1.delete()
layer2.delete()
layer3.delete()
def layer_replace(request, layername, template='layers/layer_replace.html'):
layer = _resolve_layer(
request,
layername,
'base.change_resourcebase',
_PERMISSION_MSG_MODIFY)
if request.method == 'GET':
ctx = {
'charsets': CHARSETS,
'layer': layer,
'is_featuretype': layer.is_vector(),
'is_layer': True,
}
return render_to_response(template,
RequestContext(request, ctx))
elif request.method == 'POST':
form = LayerUploadForm(request.POST, request.FILES)
tempdir = None
out = {}
if form.is_valid():
try:
tempdir, base_file = form.write_files()
if layer.is_vector() and is_raster(base_file):
out['success'] = False
out['errors'] = _("You are attempting to replace a vector layer with a raster.")
elif (not layer.is_vector()) and is_vector(base_file):
out['success'] = False
out['errors'] = _("You are attempting to replace a raster layer with a vector.")
else:
saved_layer = file_upload(
base_file,
name=layer.name,
user=request.user,
overwrite=True,
charset=form.cleaned_data["charset"],
)
out['success'] = True
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
except Exception as e:
out['success'] = False
out['errors'] = str(e)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
errormsgs = []
for e in form.errors.values():
errormsgs.append([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 400
return HttpResponse(
json.dumps(out),
mimetype='application/json',
status=status_code)
def test_layer_replace(self):
"""Test layer replace functionality
"""
vector_file = os.path.join(
gisdata.VECTOR_DATA, 'san_andres_y_providencia_administrative.shp')
vector_layer = file_upload(vector_file, overwrite=True)
raster_file = os.path.join(gisdata.RASTER_DATA, 'test_grid.tif')
raster_layer = file_upload(raster_file, overwrite=True)
self.client.login(username='******', password='******')
# test the program can determine the original layer in raster type
raster_replace_url = reverse('layer_replace',
args=[raster_layer.service_typename])
response = self.client.get(raster_replace_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(response.context['is_featuretype'], False)
# test the program can determine the original layer in vector type
vector_replace_url = reverse('layer_replace',
args=[vector_layer.service_typename])
response = self.client.get(vector_replace_url)
self.assertEquals(response.status_code, 200)
self.assertEquals(response.context['is_featuretype'], True)
# test replace a vector with a raster
response = self.client.post(vector_replace_url,
{'base_file': open(raster_file, 'rb')})
# TODO: This should really return a 400 series error with the json dict
self.assertEquals(response.status_code, 400)
response_dict = json.loads(response.content)
self.assertEquals(response_dict['success'], False)
# test replace a vector with a different vector
new_vector_file = os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
layer_path, __ = os.path.splitext(new_vector_file)
layer_base = open(layer_path + '.shp', 'rb')
layer_dbf = open(layer_path + '.dbf', 'rb')
layer_shx = open(layer_path + '.shx', 'rb')
layer_prj = open(layer_path + '.prj', 'rb')
response = self.client.post(
vector_replace_url, {
'base_file': layer_base,
'dbf_file': layer_dbf,
'shx_file': layer_shx,
'prj_file': layer_prj
})
self.assertEquals(response.status_code, 200)
response_dict = json.loads(response.content)
self.assertEquals(response_dict['success'], True)
# Get a Layer object for the newly created layer.
new_vector_layer = Layer.objects.get(pk=vector_layer.pk)
# FIXME(Ariel): Check the typename does not change.
# Test the replaced layer is indeed different from the original layer
self.assertNotEqual(vector_layer.bbox_x0, new_vector_layer.bbox_x0)
self.assertNotEqual(vector_layer.bbox_x1, new_vector_layer.bbox_x1)
self.assertNotEqual(vector_layer.bbox_y0, new_vector_layer.bbox_y0)
self.assertNotEqual(vector_layer.bbox_y1, new_vector_layer.bbox_y1)
# test an invalid user without layer replace permission
self.client.logout()
self.client.login(username='******', password='******')
response = self.client.post(
vector_replace_url, {
'base_file': layer_base,
'dbf_file': layer_dbf,
'shx_file': layer_shx,
'prj_file': layer_prj
})
self.assertEquals(response.status_code, 401)
def test_unpublished(self):
"""Test permissions on an unpublished layer
"""
thefile = os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_highway.shp')
layer = file_upload(thefile, overwrite=True)
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
try:
# request getCapabilities: layer must be there as it is published and
# advertised: we need to check if in response there is
# <Name>geonode:san_andres_y_providencia_water</Name>
geoserver_base_url = settings.OGC_SERVER['default']['LOCATION']
get_capabilities_url = 'ows?' \
'service=wms&version=1.3.0&request=GetCapabilities'
url = urljoin(geoserver_base_url, get_capabilities_url)
str_to_check = '<Name>geonode:san_andres_y_providencia_highway</Name>'
request = Request(url)
response = urlopen(request)
# by default the uploaded layer is published
self.assertTrue(layer.is_published, True)
self.assertTrue(
any(str_to_check in ensure_string(s)
for s in response.readlines()))
finally:
# Clean up and completely delete the layer
layer.delete()
# with settings disabled
with self.settings(RESOURCE_PUBLISHING=True):
layer = file_upload(thefile,
overwrite=True,
is_approved=False,
is_published=False)
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
try:
# by default the uploaded layer must be unpublished
self.assertEqual(layer.is_published, False)
# check the layer is not in GetCapabilities
request = Request(url)
response = urlopen(request)
# now test with published layer
layer = Layer.objects.get(pk=layer.pk)
layer.is_published = True
layer.save()
# we need some time to have the service up and running
time.sleep(20)
request = Request(url)
response = urlopen(request)
self.assertTrue(
any(str_to_check in ensure_string(s)
for s in response.readlines()))
finally:
# Clean up and completely delete the layer
layer.delete()
def test_permissions(self):
"""Test permissions on a layer
"""
# grab norman
norman = get_user_model().objects.get(username="******")
thefile = os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
layer = file_upload(thefile, overwrite=True)
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
# Set the layer private for not authenticated users
layer.set_permissions({'users': {'AnonymousUser': []}})
url = 'http://localhost:8080/geoserver/geonode/wms?' \
'LAYERS=geonode%3Asan_andres_y_providencia_poi&STYLES=' \
'&FORMAT=image%2Fpng&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap' \
'&SRS=EPSG%3A4326' \
'&BBOX=-81.394599749999,13.316009005566,' \
'-81.370560451855,13.372728455566' \
'&WIDTH=217&HEIGHT=512'
# test view_resourcebase permission on anonymous user
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertTrue(response.info().getheader('Content-Type'),
'application/vnd.ogc.se_xml;charset=UTF-8')
# test WMS with authenticated user that has not view_resourcebase:
# the layer must be not accessible (response is xml)
request = urllib2.Request(url)
base64string = base64.encodestring(
'%s:%s' % ('norman', 'norman')).replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
response = urllib2.urlopen(request)
self.assertTrue(response.info().getheader('Content-Type'),
'application/vnd.ogc.se_xml;charset=UTF-8')
# test WMS with authenticated user that has view_resourcebase: the layer
# must be accessible (response is image)
assign_perm('view_resourcebase', norman, layer.get_self_resource())
request = urllib2.Request(url)
base64string = base64.encodestring(
'%s:%s' % ('norman', 'norman')).replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
response = urllib2.urlopen(request)
self.assertTrue(response.info().getheader('Content-Type'), 'image/png')
# test change_layer_data
# would be nice to make a WFS/T request and test results, but this
# would work only on PostGIS layers
# test change_layer_style
url = 'http://localhost:8000/gs/rest/styles/san_andres_y_providencia_poi.xml'
sld = """<?xml version="1.0" encoding="UTF-8"?>
<sld:StyledLayerDescriptor xmlns:sld="http://www.opengis.net/sld"
xmlns:gml="http://www.opengis.net/gml" xmlns:ogc="http://www.opengis.net/ogc"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0.0"
xsi:schemaLocation="http://www.opengis.net/sld http://schemas.opengis.net/sld/1.0.0/StyledLayerDescriptor.xsd">
<sld:NamedLayer>
<sld:Name>geonode:san_andres_y_providencia_poi</sld:Name>
<sld:UserStyle>
<sld:Name>san_andres_y_providencia_poi</sld:Name>
<sld:Title>san_andres_y_providencia_poi</sld:Title>
<sld:IsDefault>1</sld:IsDefault>
<sld:FeatureTypeStyle>
<sld:Rule>
<sld:PointSymbolizer>
<sld:Graphic>
<sld:Mark>
<sld:Fill>
<sld:CssParameter name="fill">#8A7700
</sld:CssParameter>
</sld:Fill>
<sld:Stroke>
<sld:CssParameter name="stroke">#bbffff
</sld:CssParameter>
</sld:Stroke>
</sld:Mark>
<sld:Size>10</sld:Size>
</sld:Graphic>
</sld:PointSymbolizer>
</sld:Rule>
</sld:FeatureTypeStyle>
</sld:UserStyle>
</sld:NamedLayer>
</sld:StyledLayerDescriptor>"""
# user without change_layer_style cannot edit it
self.client.login(username='******', password='******')
response = self.client.put(url,
sld,
content_type='application/vnd.ogc.sld+xml')
self.assertEquals(response.status_code, 401)
# user with change_layer_style can edit it
assign_perm('change_layer_style', norman, layer)
response = self.client.put(url,
sld,
content_type='application/vnd.ogc.sld+xml')
self.assertEquals(response.status_code, 200)
# Clean up and completely delete the layer
layer.delete()
def test_unpublished(self):
"""Test permissions on an unpublished layer
"""
thefile = os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp')
layer = file_upload(thefile, overwrite=True)
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
# request getCapabilities: layer must be there as it is published and
# advertised: we need to check if in response there is
# <Name>geonode:san_andres_y_providencia_water</Name>
url = 'http://localhost:8080/geoserver/ows?' \
'service=wms&version=1.3.0&request=GetCapabilities'
str_to_check = '<Name>geonode:san_andres_y_providencia_poi</Name>'
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertTrue(any(str_to_check in s for s in response.readlines()))
# by default the uploaded layer is
self.assertTrue(layer.is_published, True)
# Clean up and completely delete the layer
layer.delete()
# with settings disabled
with self.settings(RESOURCE_PUBLISHING=True):
thefile = os.path.join(
gisdata.VECTOR_DATA,
'san_andres_y_providencia_administrative.shp')
layer = file_upload(thefile, overwrite=True)
layer.set_default_permissions()
check_layer(layer)
# we need some time to have the service up and running
time.sleep(20)
str_to_check = '<Name>san_andres_y_providencia_administrative</Name>'
# by default the uploaded layer must be unpublished
self.assertEqual(layer.is_published, False)
# check the layer is not in GetCapabilities
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertFalse(
any(str_to_check in s for s in response.readlines()))
# now test with published layer
resource = layer.get_self_resource()
resource.is_published = True
resource.save()
request = urllib2.Request(url)
response = urllib2.urlopen(request)
self.assertTrue(
any(str_to_check in s for s in response.readlines()))
# Clean up and completely delete the layer
layer.delete()
def testPrintProxy(self):
""" Test the PrintProxyMiddleware if activated.
It should respect the permissions on private layers.
"""
if 'geonode.middleware.PrintProxyMiddleware' in settings.MIDDLEWARE_CLASSES:
# STEP 1: Import a layer
from geonode.maps.models import Map
self.client.login(username='******', password='******')
# TODO: Would be nice to ensure the name is available before
# running the test...
norman = get_user_model().objects.get(username="******")
saved_layer = file_upload(
os.path.join(gisdata.VECTOR_DATA,
"san_andres_y_providencia_poi.shp"),
name="san_andres_y_providencia_poi_by_norman",
user=norman,
overwrite=True,
)
# Set the layer private
saved_layer.set_permissions(
{'users': {
'AnonymousUser': ['view_resourcebase']
}})
url = reverse('layer_metadata',
args=[saved_layer.service_typename])
# check is accessible while logged in
resp = self.client.get(url)
self.assertEquals(resp.status_code, 200)
# check is inaccessible when not logged in
self.client.logout()
resp = self.client.get(url)
self.assertEquals(resp.status_code, 302)
# STEP 2: Create a Map with that layer
map_obj = Map(owner=norman, zoom=0, center_x=0, center_y=0)
map_obj.create_from_layer_list(norman, [saved_layer], 'title', '')
# STEP 3: Print the map
print_url = settings.OGC_SERVER['default'][
'LOCATION'] + 'pdf/create.json'
post_payload = {
'dpi':
75,
'layers': [{
'baseURL':
settings.OGC_SERVER['default']['LOCATION'] +
'wms?SERVICE=WMS&',
'format':
"image/png",
'customParams': {
'TILED': True,
'TRANSPARENT': True
},
'layers': [saved_layer.service_typename],
'opacity':
1,
'singleTile':
False,
'type':
'WMS'
}],
'layout':
'A4 portrait',
'mapTitle':
'test',
'outputFilename':
'print',
'srs':
'EPSG:900913',
'units':
'm'
}
self.client.post(print_url, post_payload)
# Test the layer is still inaccessible as non authenticated
resp = self.client.get(url)
self.assertEquals(resp.status_code, 302)
else:
pass
def test_map_thumbnail(self):
"""Creating map will create thumbnail."""
filename = os.path.join(
gisdata.GOOD_DATA, 'raster/relief_san_andres.tif')
layer1 = file_upload(filename)
filename = os.path.join(
gisdata.GOOD_DATA,
'vector/san_andres_y_providencia_administrative.shp')
layer2 = file_upload(filename)
""":type: geonode.layers.models.Layer"""
# construct json request for new map
json_payload = InitialSetup.generate_initial_map(layer1, layer2)
self.client.login(username='******', password='******')
response = self.client.post(
reverse('new_map_json'),
json.dumps(json_payload),
content_type='application/json')
self.assertEqual(response.status_code, 200)
map_id = json.loads(response.content).get('id')
map = Map.objects.get(id=map_id)
# check that we have remote thumbnail
remote_thumbnail_link = map.link_set.filter(
name__icontains='remote thumbnail').first()
self.assertTrue(remote_thumbnail_link.url)
# thumbnail won't generate because remote thumbnail uses public
# address
remote_thumbnail_url = remote_thumbnail_link.url
# Replace url's basename, we want to access it using django client
parse_result = urlparse.urlsplit(remote_thumbnail_url)
remote_thumbnail_url = urlparse.urlunsplit(
('', '', parse_result.path, parse_result.query, ''))
response = self.client.get(remote_thumbnail_url)
thumbnail_dir = os.path.join(settings.MEDIA_ROOT, 'thumbs')
thumbnail_path = os.path.join(thumbnail_dir, 'map-thumb.png')
map.save_thumbnail(thumbnail_path, response.content)
# Check thumbnail created
self.assertTrue(os.path.exists(thumbnail_path))
self.assertEqual(what(thumbnail_path), 'png')
# Check that now we have thumbnail
self.assertTrue(map.has_thumbnail())
missing_thumbnail_url = staticfiles.static(settings.MISSING_THUMBNAIL)
self.assertTrue(map.get_thumbnail_url() != missing_thumbnail_url)
thumbnail_links = map.link_set.filter(name__icontains='thumbnail')
self.assertTrue(len(thumbnail_links) > 0)
link_names = ['remote thumbnail', 'thumbnail']
for link in thumbnail_links:
self.assertIn(link.name.lower(), link_names)
# cleanup
map.delete()
layer1.delete()
layer2.delete()
def test_inasafe_metadata_capture(self):
"""Test capturing InaSAFE metadata from layer's metadata."""
data_helper = self.data_helper
"""Case 1. Uploading layer that have no InaSAFE metadata"""
# Uploading base file only should generate basic metadata.
ash_fall_only = data_helper.misc('ash_fall_base_only.tif')
ash_fall_only_layer = file_upload(ash_fall_only)
# There should be no metadata processing so we can just inspect the
# layer
metadata_link = ash_fall_only_layer.link_set.metadata().get(name='ISO')
response = self.client.get(metadata_link.url)
self.assertEqual(response.status_code, 200)
metadata_xml = response.content
# This should be a valid xml
namespaces = ISO_METADATA_NAMESPACES
root_metadata_xml = etree.fromstring(metadata_xml)
identification_info = root_metadata_xml.xpath(
'//gmd:identificationInfo', namespaces=namespaces)
self.assertTrue(identification_info)
inasafe_keywords = root_metadata_xml.xpath(
ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)
# Will not find inasafe keywords safely
self.assertFalse(inasafe_keywords)
self.assertTrue(metadata_xml)
"""Case 2. Uploading layer that have InaSAFE metadata"""
# Upload layer with metadata
ash_fall = data_helper.hazard('ash_fall.tif')
ash_fall_layer = file_upload(ash_fall)
# Wait for InaSAFE to process metadata
wait_metadata(ash_fall_layer)
# Validate internal model
ash_fall_layer = Layer.objects.get(id=ash_fall_layer.id)
self.assertTrue(ash_fall_layer.inasafe_metadata)
self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_xml)
self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_json)
self.assertTrue(ash_fall_layer.inasafe_metadata.keywords)
self.assertEqual(ash_fall_layer.inasafe_metadata.layer_purpose,
'hazard')
self.assertEqual(ash_fall_layer.inasafe_metadata.category,
'volcanic_ash')
# Fetch metadata from URL
metadata_link = ash_fall_layer.link_set.metadata().get(name='ISO')
response = self.client.get(metadata_link.url)
self.assertEqual(response.status_code, 200)
metadata_xml = response.content
# This should be a valid xml
namespaces = ISO_METADATA_NAMESPACES
root_metadata_xml = etree.fromstring(metadata_xml)
identification_info = root_metadata_xml.xpath(
'//gmd:identificationInfo', namespaces=namespaces)
self.assertTrue(identification_info)
inasafe_keywords = root_metadata_xml.xpath(
ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)
# Now it should find InaSAFE metadata
self.assertTrue(inasafe_keywords)
# Xml in metadata page and model should be the same structurally
pattern = r'<inasafe [^<]*>'
replace = '<inasafe xmlns:gmd="http://www.isotc211.org/2005/gmd" ' \
'xmlns:gco="http://www.isotc211.org/2005/gco">'
link_xml = etree.tostring(inasafe_keywords[0],
pretty_print=True,
xml_declaration=False)
model_xml = etree.tostring(etree.fromstring(
ash_fall_layer.inasafe_metadata.keywords_xml),
pretty_print=True,
xml_declaration=False)
# We replace XML header so it will have the same header
link_xml = re.sub(pattern, replace, link_xml).strip()
model_xml = re.sub(pattern, replace, model_xml).strip()
# Match XML representations
self.assertXMLEqual(link_xml, model_xml)
# QGIS XML file should have matched string representation
with codecs.open(ash_fall_layer.qgis_layer.xml_path,
encoding='utf-8') as f:
qgis_xml = f.read()
self.assertEqual(ash_fall_layer.metadata_xml, qgis_xml)
"""Case 3. Test that uploading new metadata will also update
InaSAFE metadata. It will be gone if there is no InaSAFE metadata
"""
# Upload metadata with no InaSAFE keywords
previous_inasafe_keywords = inasafe_keywords[0]
inasafe_keywords[0].getparent().remove(inasafe_keywords[0])
__, metadata_filename = tempfile.mkstemp('.xml')
with codecs.open(metadata_filename, mode='w', encoding='utf-8') as f:
f.write(etree.tostring(root_metadata_xml))
ash_fall_layer = file_upload(metadata_filename,
metadata_upload_form=True)
# We do not wait for InaSAFE to process metadata
# Supposedly, there's no metadata
# Validate internal model
ash_fall_layer = Layer.objects.get(id=ash_fall_layer.id)
self.assertTrue(ash_fall_layer.inasafe_metadata)
# Now there is no InaSAFE keywords
self.assertFalse(ash_fall_layer.inasafe_metadata.keywords_xml)
self.assertFalse(ash_fall_layer.inasafe_metadata.keywords_json)
self.assertFalse(ash_fall_layer.inasafe_metadata.keywords)
self.assertFalse(ash_fall_layer.inasafe_metadata.layer_purpose)
self.assertFalse(ash_fall_layer.inasafe_metadata.category)
# Metadata download will not show InaSAFE keywords
metadata_link = ash_fall_layer.link_set.metadata().get(name='ISO')
response = self.client.get(metadata_link.url)
self.assertEqual(response.status_code, 200)
metadata_xml = response.content
root_metadata_xml = etree.fromstring(metadata_xml)
inasafe_keywords = root_metadata_xml.xpath(
ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)
self.assertFalse(inasafe_keywords)
# QGIS XML file should have matched string representation
with codecs.open(ash_fall_layer.qgis_layer.xml_path,
encoding='utf-8') as f:
qgis_xml = f.read()
self.assertEqual(ash_fall_layer.metadata_xml, qgis_xml)
"""Case 4. Test that uploading new metadata will also update
InaSAFE metadata. Upload new metadata with InaSAFE keywords.
"""
# Change some detail in the metadata
inasafe_keywords = previous_inasafe_keywords
keyword_version = inasafe_keywords.xpath(
'keyword_version/gco:CharacterString', namespaces=namespaces)[0]
self.assertEqual(keyword_version.text, '4.3')
# Change keyword version
keyword_version.text = '4.4'
# Attach InaSAFE keywords to the new metadata
supplemental_info = root_metadata_xml.xpath(
'//gmd:supplementalInformation', namespaces=namespaces)[0]
supplemental_info.append(inasafe_keywords)
# Upload metadata with new InaSAFE keywords
with codecs.open(metadata_filename, mode='w', encoding='utf-8') as f:
f.write(etree.tostring(root_metadata_xml))
ash_fall_layer = file_upload(metadata_filename,
metadata_upload_form=True)
# Wait for InaSAFE to process metadata
wait_metadata(ash_fall_layer)
# Validate internal model
ash_fall_layer = Layer.objects.get(id=ash_fall_layer.id)
self.assertTrue(ash_fall_layer.inasafe_metadata)
self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_xml)
self.assertTrue(ash_fall_layer.inasafe_metadata.keywords_json)
self.assertTrue(ash_fall_layer.inasafe_metadata.keywords)
self.assertEqual(ash_fall_layer.inasafe_metadata.layer_purpose,
'hazard')
self.assertEqual(ash_fall_layer.inasafe_metadata.category,
'volcanic_ash')
# Fetch metadata from URL
metadata_link = ash_fall_layer.link_set.metadata().get(name='ISO')
response = self.client.get(metadata_link.url)
self.assertEqual(response.status_code, 200)
metadata_xml = response.content
# This should be a valid xml
namespaces = ISO_METADATA_NAMESPACES
root_metadata_xml = etree.fromstring(metadata_xml)
inasafe_keywords = root_metadata_xml.xpath(
ISO_METADATA_INASAFE_KEYWORD_TAG, namespaces=namespaces)
# Now it should find InaSAFE metadata
self.assertTrue(inasafe_keywords)
inasafe_keywords = inasafe_keywords[0]
# Check new keywords version
keyword_version = inasafe_keywords.xpath(
'//keyword_version/gco:CharacterString', namespaces=namespaces)[0]
self.assertEqual(keyword_version.text, '4.4')
# QGIS XML file should have matched string representation
with codecs.open(ash_fall_layer.qgis_layer.xml_path,
encoding='utf-8') as f:
qgis_xml = f.read()
self.assertEqual(ash_fall_layer.metadata_xml, qgis_xml)
os.remove(metadata_filename)
ash_fall_only_layer.delete()
ash_fall_layer.delete()
def test_add_delete_style(self):
"""Test add new style using qgis_server views."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
layer = file_upload(filename)
""":type: geonode.layers.models.Layer"""
self.client.login(username='******', password='******')
qml_path = self.data_path('test_grid.qml')
add_style_url = reverse(
'qgis_server:upload-qml',
kwargs={
'layername': layer.name})
with open(qml_path) as file_handle:
form_data = {
'name': 'new_style',
'title': 'New Style',
'qml': file_handle
}
response = self.client.post(
add_style_url,
data=form_data)
self.assertEqual(response.status_code, 201)
actual_list_style = style_list(layer, internal=False)
if actual_list_style:
expected_list_style = ['default', 'new_style']
self.assertEqual(
set(expected_list_style),
set([style.name for style in actual_list_style]))
# Test delete request
delete_style_url = reverse(
'qgis_server:remove-qml',
kwargs={
'layername': layer.name,
'style_name': 'default'})
response = self.client.delete(delete_style_url)
self.assertEqual(response.status_code, 200)
actual_list_style = style_list(layer, internal=False)
if actual_list_style:
expected_list_style = ['new_style']
self.assertEqual(
set(expected_list_style),
set([style.name for style in actual_list_style]))
# Check new default
default_style_url = reverse(
'qgis_server:default-qml',
kwargs={
'layername': layer.name})
response = self.client.get(default_style_url)
self.assertEqual(response.status_code, 200)
expected_default_style_retval = {
'name': 'new_style',
}
actual_default_style_retval = json.loads(response.content)
for key, value in expected_default_style_retval.iteritems():
self.assertEqual(actual_default_style_retval[key], value)
layer.delete()
def test_layercount_strategy(self):
"""
Test layercount SHARD_STRATEGY.
"""
settings.SHARD_STRATEGY = 'layercount'
owner = get_user_model().objects.get(username="******")
layers_to_upload = ('layer_01', 'layer_02', 'layer_03', 'layer_04',
'layer_05')
for layer in layers_to_upload:
print 'Uploading layer %s...' % layer
saved_layer = file_upload(
os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp'),
name=layer,
user=owner,
overwrite=True,
)
saved_layer.set_default_permissions()
# check layers in geoserver if they have correct store type and datasource
cat = gs_catalog
i = 0
for layer in layers_to_upload:
i += 1
gs_layer = cat.get_layer(layer)
self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
if i in (1, 2):
self.assertEqual(gs_layer.resource.store.name,
'%s00000' % SHARD_PREFIX)
if i in (3, 4):
self.assertEqual(gs_layer.resource.store.name,
'%s00001' % SHARD_PREFIX)
if i == 5:
self.assertEqual(gs_layer.resource.store.name,
'%s00002' % SHARD_PREFIX)
# remove one layer and see what happens when adding a new one
layer = Layer.objects.get(name='layer_05')
layer.delete()
saved_layer = file_upload(
os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp'),
name='layer_06',
user=owner,
overwrite=True,
)
saved_layer.set_default_permissions()
gs_layer = cat.get_layer('layer_06')
self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
self.assertEqual(gs_layer.resource.store.name,
'%s00002' % SHARD_PREFIX)
# check Database objects and layers_count
self.assertEqual(
Database.objects.filter(name__contains=('%s0000' %
SHARD_PREFIX)).count(), 3)
self.assertEqual(
Database.objects.get(name='%s00000' % SHARD_PREFIX).layers_count,
2)
self.assertEqual(
Database.objects.get(name='%s00001' % SHARD_PREFIX).layers_count,
2)
self.assertEqual(
Database.objects.get(name='%s00002' % SHARD_PREFIX).layers_count,
1)
# remove layers
layers_to_delete = ('layer_01', 'layer_02', 'layer_03', 'layer_04',
'layer_06')
for layer_name in layers_to_delete:
layer = Layer.objects.get(name=layer_name)
layer.delete()
for i in (0, 1, 2):
self.assertEqual(
Database.objects.get(name='%s0000%s' %
(SHARD_PREFIX, i)).layers_count, 0)
def add_wfs_layer(
endpoint,
version,
typename,
metadata_string=None,
title=None,
bbox=None,
user=None,
password=None):
endpoint_parsed = urlparse.urlparse(endpoint)
q_dict = {
'version': version,
'typename': typename,
'outputFormat': 'shape-zip',
'request': 'GetFeature',
'service': 'WFS',
'srsName': 'EPSG:4326',
}
if bbox:
q_dict['bbox'] = ','.join(bbox)
parsed_url = urlparse.ParseResult(
scheme=endpoint_parsed.scheme,
netloc=endpoint_parsed.netloc,
path=endpoint_parsed.path,
params=None,
query=urllib.urlencode(q_dict),
fragment=None
)
tmpfile = download_file(parsed_url.geturl(), user=user, password=password)
# args = [
# 'ogr2ogr',
# '-nlt POLYGON',
# '-skipfailures',
# '%s.shp' % tmpfile,
# tmpfile,
# 'OGRGeoJSON'
# ]
#
# retval = subprocess.call(args)
# # get metadata file
# if metadata_string:
# if not isinstance(metadata_string, unicode):
# metadata_string = unicode(metadata_string, 'utf-8')
# metadata_file = '%s.xml' % tmpfile
# with io.open(metadata_file, mode='w', encoding='utf-8') as f:
# f.write(metadata_string)
#
# saved_layer = None
# if retval == 0:
# saved_layer = file_upload(
# '%s.shp' % tmpfile,
# overwrite=True)
# saved_layer.set_default_permissions()
# saved_layer.title = title or typename
# saved_layer.save()
#
# # cleanup
# dir_name = os.path.dirname(tmpfile)
# for root, dirs, files in os.walk(dir_name):
# for f in files:
# if tmpfile in f:
# try:
# os.remove(os.path.join(root, f))
# except:
# pass
dir_name = os.path.dirname(tmpfile)
saved_layer = None
metadata_file = None
with ZipFile(tmpfile) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if '.shp' in ext:
# get metadata file
if metadata_string:
if not isinstance(metadata_string, unicode):
metadata_string = unicode(metadata_string, 'utf-8')
metadata_file = '%s.xml' % basename
metadata_file = os.path.join(dir_name, metadata_file)
metadata_string = cleanup_metadata(metadata_string)
with io.open(metadata_file, mode='w',
encoding='utf-8') as f:
f.write(metadata_string)
# process shapefile layer
saved_layer = file_upload(
os.path.join(dir_name, name),
overwrite=True)
saved_layer.set_default_permissions()
saved_layer.title = title or typename
saved_layer.save()
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except:
pass
if metadata_file:
try:
os.remove(metadata_file)
except:
pass
# cleanup
try:
os.remove(tmpfile)
except:
pass
return saved_layer
def test_monthly_strategy(self, mock_obj):
"""
Test monthly SHARD_STRATEGY.
"""
settings.SHARD_STRATEGY = 'monthly'
# testing if layer is created in 197102 shard
global MONTH
MONTH = 2
owner = get_user_model().objects.get(username="******")
saved_layer = file_upload(
os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp'),
name='layer_a',
user=owner,
overwrite=True,
)
saved_layer.set_default_permissions()
cat = gs_catalog
gs_layer = cat.get_layer('layer_a')
self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
self.assertEqual(gs_layer.resource.store.name,
'%s197102' % SHARD_PREFIX)
# afer one month
# testing if layer is created in 197103 shard
global MONTH
MONTH = 3
saved_layer = file_upload(
os.path.join(gisdata.VECTOR_DATA,
'san_andres_y_providencia_poi.shp'),
name='layer_b',
user=owner,
overwrite=True,
)
saved_layer.set_default_permissions()
gs_layer = cat.get_layer('layer_b')
self.assertEqual(gs_layer.resource.store.type, 'PostGIS')
self.assertEqual(gs_layer.resource.store.name,
'%s197103' % SHARD_PREFIX)
# check Database objects
self.assertEqual(
Database.objects.filter(name__contains=('%s1971' %
SHARD_PREFIX)).count(), 2)
self.assertEqual(
Database.objects.get(name='%s197102' % SHARD_PREFIX).layers_count,
1)
self.assertEqual(
Database.objects.get(name='%s197103' % SHARD_PREFIX).layers_count,
1)
# remove the layers and see what happens
for layer_name in ('layer_a', 'layer_b'):
layer = Layer.objects.get(name=layer_name)
layer.delete()
self.assertEqual(
Database.objects.get(name='%s197102' % SHARD_PREFIX).layers_count,
0)
self.assertEqual(
Database.objects.get(name='%s197103' % SHARD_PREFIX).layers_count,
0)
def test_style_management_url(self):
"""Test QGIS Server style management url construction."""
filename = os.path.join(gisdata.GOOD_DATA, 'raster/test_grid.tif')
uploaded = file_upload(filename)
# Get default style
# There will always be a default style when uploading a layer
style_url = style_get_url(uploaded, 'default', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers.get('Content-Type'), 'text/xml')
# it has to contains qgis tags
style_xml = dlxml.fromstring(response.content)
self.assertTrue('qgis' in style_xml.tag)
# Add new style
# change default style slightly
self.assertTrue('WhiteToBlack' not in response.content)
self.assertTrue('BlackToWhite' in response.content)
new_style_xml = dlxml.fromstring(
response.content.replace('BlackToWhite', 'WhiteToBlack'))
new_xml_content = etree.tostring(new_style_xml, pretty_print=True)
# save it to qml file, accessible by qgis server
qgis_layer = QGISServerLayer.objects.get(layer=uploaded)
with open(qgis_layer.qml_path, mode='w') as f:
f.write(new_xml_content)
style_url = style_add_url(uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, 'OK')
# Get style list
qml_styles = style_list(uploaded, internal=False)
if qml_styles:
expected_style_names = ['default', 'new_style']
actual_style_names = [s.name for s in qml_styles]
self.assertEqual(
set(expected_style_names),
set(actual_style_names))
# Get new style
style_url = style_get_url(uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers.get('Content-Type'), 'text/xml')
self.assertTrue('WhiteToBlack' in response.content)
# Set default style
style_url = style_set_default_url(
uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, 'OK')
# Remove style
style_url = style_remove_url(uploaded, 'new_style', internal=True)
response = requests.get(style_url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, 'OK')
# Cleanup
uploaded.delete()
def layer_upload(request, template='upload/layer_upload.html'):
if request.method == 'GET':
ctx = {'charsets': CHARSETS}
return render_to_response(template, RequestContext(request, ctx))
elif request.method == 'POST':
form = NewLayerUploadForm(request.POST, request.FILES)
tempdir = None
errormsgs = []
out = {'success': False}
if form.is_valid():
title = form.cleaned_data["layer_title"]
# Replace dots in filename - GeoServer REST API upload bug
# and avoid any other invalid characters.
# Use the title if possible, otherwise default to the filename
if title is not None and len(title) > 0:
name_base = title
else:
name_base, __ = os.path.splitext(
form.cleaned_data["base_file"].name)
name = slugify(name_base.replace(".", "_"))
try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
# This should be followed up in upstream Django.
tempdir, base_file = form.write_files()
saved_layer = file_upload(
base_file,
name=name,
user=request.user,
overwrite=False,
charset=form.cleaned_data["charset"],
abstract=form.cleaned_data["abstract"],
title=form.cleaned_data["layer_title"],
)
except Exception as e:
logger.exception(e)
out['success'] = False
out['errors'] = str(e)
else:
out['success'] = True
out['url'] = reverse('layer_detail',
args=[saved_layer.service_typename])
permissions = form.cleaned_data["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
for e in form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 500
return HttpResponse(json.dumps(out),
mimetype='application/json',
status=status_code)
def _save_layer(layer_dict,
base_file,
user,
tempdir=None,
overwrite=False,
keywords=[]):
"""Thin wrapper around file_upload."""
out = {}
title = layer_dict.get("layer_title", "")
# Replace dots in filename - GeoServer REST API upload bug
# and avoid any other invalid characters.
# Use the title if possible, otherwise default to the filename
if title is not None and len(title) > 0:
name_base = title
else:
name_base, __ = os.path.splitext(base_file)
name = slugify(name_base.replace(".", "_"))
try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
# This should be followed up in upstream Django.
saved_layer = file_upload(base_file,
name=name,
user=user,
style_name=layer_dict.get(
"style_name", None),
overwrite=overwrite,
charset=layer_dict.get("charset", "UTF-8"),
abstract=layer_dict.get("abstract", ""),
title=title,
keywords=layer_dict.get("keywords", []),
date=layer_dict.get("date", None))
except Exception as e:
exception_type, error, tb = sys.exc_info()
logger.exception(e)
out['success'] = False
out['errors'] = str(error)
# Assign the error message to the latest UploadSession from that user.
latest_uploads = UploadSession.objects.filter(
user=user).order_by('-date')
if latest_uploads.count() > 0:
upload_session = latest_uploads[0]
upload_session.error = str(error)
upload_session.traceback = traceback.format_exc(tb)
upload_session.context = log_snippet(CONTEXT_LOG_FILE)
upload_session.save()
out['traceback'] = upload_session.traceback
out['context'] = upload_session.context
out['upload_session'] = upload_session.id
else:
out['success'] = True
if hasattr(saved_layer, 'info'):
out['info'] = saved_layer.info
out['url'] = reverse('layer_detail',
args=[saved_layer.service_typename])
out["layer"] = saved_layer
upload_session = saved_layer.upload_session
upload_session.processed = True
upload_session.save()
permissions = layer_dict["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
return out
def test_run_analysis_no_aggregation(self):
"""Test running analysis without aggregation."""
data_helper = InaSAFETestData()
flood = data_helper.hazard('flood_data.geojson')
buildings = data_helper.exposure('buildings.geojson')
flood_layer = file_upload(flood)
# Wait until metadata is read
wait_metadata(flood_layer)
buildings_layer = file_upload(buildings)
# Wait until metadata is read
wait_metadata(buildings_layer)
# Check layer uploaded
self.assertEqual(Layer.objects.filter(id=flood_layer.id).count(), 1)
self.assertEqual(
Layer.objects.filter(id=buildings_layer.id).count(), 1)
form_data = {
'user': AnonymousUser(),
'user_title': 'Flood on Buildings custom Analysis',
'exposure_layer': buildings_layer.id,
'hazard_layer': flood_layer.id,
'keep': False,
'extent_option': Analysis.HAZARD_EXPOSURE_CODE
}
form = AnalysisCreationForm(form_data, user=form_data['user'])
if not form.is_valid():
LOGGER.debug(form.errors)
self.assertTrue(form.is_valid())
analysis = form.save()
""":type: geosafe.models.Analysis"""
while analysis.get_task_state() == 'PENDING':
analysis.refresh_from_db()
time.sleep(1)
if analysis.get_task_result().failed():
LOGGER.debug(analysis.get_task_result().traceback)
self.assertTrue(analysis.get_task_result().successful())
self.assertEqual(analysis.get_task_state(), 'SUCCESS')
while not analysis.impact_layer:
analysis.refresh_from_db()
time.sleep(1)
impact_layer = analysis.impact_layer
wait_metadata(impact_layer)
LOGGER.debug('Layers: {0}'.format(Layer.objects.all()))
self.assertEqual(impact_layer.inasafe_metadata.layer_purpose,
'impact_analysis')
flood_layer.delete()
buildings_layer.delete()
impact_layer.delete()
def layer_upload(request, template='upload/layer_upload.html'):
if request.method == 'GET':
mosaics = Layer.objects.filter(is_mosaic=True).order_by('name')
ctx = {
'mosaics': mosaics,
'charsets': CHARSETS,
'is_layer': True,
}
return render_to_response(template, RequestContext(request, ctx))
elif request.method == 'POST':
form = NewLayerUploadForm(request.POST, request.FILES)
tempdir = None
errormsgs = []
out = {'success': False}
if form.is_valid():
title = form.cleaned_data["layer_title"]
# Replace dots in filename - GeoServer REST API upload bug
# and avoid any other invalid characters.
# Use the title if possible, otherwise default to the filename
if title is not None and len(title) > 0:
name_base = title
else:
name_base, __ = os.path.splitext(
form.cleaned_data["base_file"].name)
name = slugify(name_base.replace(".", "_"))
try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
# This should be followed up in upstream Django.
tempdir, base_file = form.write_files()
saved_layer = file_upload(
base_file,
name=name,
user=request.user,
overwrite=False,
charset=form.cleaned_data["charset"],
#abstract=form.cleaned_data["abstract"],
abstract=" ",
title=form.cleaned_data["layer_title"],
metadata_uploaded_preserve=form.
cleaned_data["metadata_uploaded_preserve"])
except Exception as e:
exception_type, error, tb = sys.exc_info()
logger.exception(e)
out['success'] = False
out['errors'] = str(error)
# Assign the error message to the latest UploadSession from that user.
latest_uploads = UploadSession.objects.filter(
user=request.user).order_by('-date')
if latest_uploads.count() > 0:
upload_session = latest_uploads[0]
upload_session.error = str(error)
upload_session.traceback = traceback.format_exc(tb)
upload_session.context = log_snippet(CONTEXT_LOG_FILE)
upload_session.save()
out['traceback'] = upload_session.traceback
out['context'] = upload_session.context
out['upload_session'] = upload_session.id
else:
out['success'] = True
if hasattr(saved_layer, 'info'):
out['info'] = saved_layer.info
out['url'] = reverse('layer_detail',
args=[saved_layer.service_typename])
upload_session = saved_layer.upload_session
upload_session.processed = True
upload_session.save()
permissions = form.cleaned_data["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
for e in form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 400
return HttpResponse(json.dumps(out),
content_type='application/json',
status=status_code)
def layer_upload(request, template='upload/layer_upload.html'):
if request.method == 'GET':
mosaics = Layer.objects.filter(is_mosaic=True).order_by('name')
ctx = {
'mosaics': mosaics,
'charsets': CHARSETS,
'is_layer': True,
}
return render_to_response(template, RequestContext(request, ctx))
elif request.method == 'POST':
form = NewLayerUploadForm(request.POST, request.FILES)
tempdir = None
saved_layer = None
errormsgs = []
out = {'success': False}
if form.is_valid():
title = form.cleaned_data["layer_title"]
# Replace dots in filename - GeoServer REST API upload bug
# and avoid any other invalid characters.
# Use the title if possible, otherwise default to the filename
if title is not None and len(title) > 0:
name_base = title
else:
name_base, __ = os.path.splitext(
form.cleaned_data["base_file"].name)
title = slugify(name_base.replace(".", "_"))
name = slugify(name_base.replace(".", "_"))
if form.cleaned_data["abstract"] is not None and len(form.cleaned_data["abstract"]) > 0:
abstract = form.cleaned_data["abstract"]
else:
abstract = "No abstract provided."
try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
# This should be followed up in upstream Django.
tempdir, base_file = form.write_files()
if not form.cleaned_data["style_upload_form"]:
saved_layer = file_upload(
base_file,
name=name,
user=request.user,
overwrite=False,
charset=form.cleaned_data["charset"],
abstract=abstract,
title=title,
metadata_uploaded_preserve=form.cleaned_data[
"metadata_uploaded_preserve"],
metadata_upload_form=form.cleaned_data["metadata_upload_form"])
else:
saved_layer = Layer.objects.get(alternate=title)
if not saved_layer:
msg = 'Failed to process. Could not find matching layer.'
raise Exception(msg)
sld = open(base_file).read()
# Check SLD is valid
extract_name_from_sld(gs_catalog, sld, sld_file=base_file)
match = None
styles = list(saved_layer.styles.all()) + [
saved_layer.default_style]
for style in styles:
if style and style.name == saved_layer.name:
match = style
break
cat = gs_catalog
layer = cat.get_layer(title)
if match is None:
try:
cat.create_style(saved_layer.name, sld, raw=True, workspace=settings.DEFAULT_WORKSPACE)
style = cat.get_style(saved_layer.name, workspace=settings.DEFAULT_WORKSPACE) or \
cat.get_style(saved_layer.name)
if layer and style:
layer.default_style = style
cat.save(layer)
saved_layer.default_style = save_style(style)
except Exception as e:
logger.exception(e)
else:
style = cat.get_style(saved_layer.name, workspace=settings.DEFAULT_WORKSPACE) or \
cat.get_style(saved_layer.name)
# style.update_body(sld)
try:
cat.create_style(saved_layer.name, sld, overwrite=True, raw=True,
workspace=settings.DEFAULT_WORKSPACE)
style = cat.get_style(saved_layer.name, workspace=settings.DEFAULT_WORKSPACE) or \
cat.get_style(saved_layer.name)
if layer and style:
layer.default_style = style
cat.save(layer)
saved_layer.default_style = save_style(style)
except Exception as e:
logger.exception(e)
# Invalidate GeoWebCache for the updated resource
_invalidate_geowebcache_layer(saved_layer.alternate)
except Exception as e:
exception_type, error, tb = sys.exc_info()
logger.exception(e)
out['success'] = False
out['errors'] = str(error)
# Assign the error message to the latest UploadSession from
# that user.
latest_uploads = UploadSession.objects.filter(
user=request.user).order_by('-date')
if latest_uploads.count() > 0:
upload_session = latest_uploads[0]
upload_session.error = str(error)
upload_session.traceback = traceback.format_exc(tb)
upload_session.context = log_snippet(CONTEXT_LOG_FILE)
upload_session.save()
out['traceback'] = upload_session.traceback
out['context'] = upload_session.context
out['upload_session'] = upload_session.id
else:
out['success'] = True
if hasattr(saved_layer, 'info'):
out['info'] = saved_layer.info
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
if hasattr(saved_layer, 'bbox_string'):
out['bbox'] = saved_layer.bbox_string
if hasattr(saved_layer, 'srid'):
out['crs'] = {
'type': 'name',
'properties': saved_layer.srid
}
out['ogc_backend'] = settings.OGC_SERVER['default']['BACKEND']
upload_session = saved_layer.upload_session
if upload_session:
upload_session.processed = True
upload_session.save()
permissions = form.cleaned_data["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
saved_layer.handle_moderated_uploads()
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
for e in form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 400
if settings.MONITORING_ENABLED:
request.add_resource('layer', saved_layer.alternate if saved_layer else name)
return HttpResponse(
json.dumps(out),
content_type='application/json',
status=status_code)
def layer_replace(request, layername, template='layers/layer_replace.html'):
layer = _resolve_layer(
request,
layername,
'base.change_resourcebase',
_PERMISSION_MSG_MODIFY)
if request.method == 'GET':
ctx = {
'charsets': CHARSETS,
'layer': layer,
'is_featuretype': layer.is_vector(),
'is_layer': True,
}
return render_to_response(template,
RequestContext(request, ctx))
elif request.method == 'POST':
form = LayerUploadForm(request.POST, request.FILES)
tempdir = None
out = {}
if form.is_valid():
try:
tempdir, base_file = form.write_files()
if layer.is_vector() and is_raster(base_file):
out['success'] = False
out['errors'] = _(
"You are attempting to replace a vector layer with a raster.")
elif (not layer.is_vector()) and is_vector(base_file):
out['success'] = False
out['errors'] = _(
"You are attempting to replace a raster layer with a vector.")
else:
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
# delete geoserver's store before upload
cat = gs_catalog
cascading_delete(cat, layer.typename)
out['ogc_backend'] = geoserver.BACKEND_PACKAGE
elif check_ogc_backend(qgis_server.BACKEND_PACKAGE):
try:
qgis_layer = QGISServerLayer.objects.get(
layer=layer)
qgis_layer.delete()
except QGISServerLayer.DoesNotExist:
pass
out['ogc_backend'] = qgis_server.BACKEND_PACKAGE
saved_layer = file_upload(
base_file,
name=layer.name,
user=request.user,
overwrite=True,
charset=form.cleaned_data["charset"],
)
out['success'] = True
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
except Exception as e:
logger.exception(e)
tb = traceback.format_exc()
out['success'] = False
out['errors'] = str(tb)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
errormsgs = []
for e in form.errors.values():
errormsgs.append([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 400
return HttpResponse(
json.dumps(out),
content_type='application/json',
status=status_code)
def layer_upload(request, template='upload/layer_upload.html'):
if request.method == 'GET':
mosaics = Layer.objects.filter(is_mosaic=True).order_by('name')
ctx = {
'mosaics':
mosaics,
'charsets':
CHARSETS,
'is_layer':
True,
'allowed_file_types': ['.cst', '.dbf', '.prj', '.shp', '.shx'],
'categories':
TopicCategory.objects.all(),
'organizations':
GroupProfile.objects.filter(groupmember__user=request.user),
'user_organization':
GroupProfile.objects.filter(
groupmember__user=request.user).first()
}
return render_to_response(template, RequestContext(request, ctx))
elif request.method == 'POST':
form = NewLayerUploadForm(request.POST, request.FILES)
tempdir = None
errormsgs = []
out = {'success': False}
if form.is_valid():
title = form.cleaned_data["layer_title"]
category = form.cleaned_data["category"]
organization_id = form.cleaned_data["organization"]
admin_upload = form.cleaned_data["admin_upload"]
group = GroupProfile.objects.get(id=organization_id)
# Replace dots in filename - GeoServer REST API upload bug
# and avoid any other invalid characters.
# Use the title if possible, otherwise default to the filename
if title is not None and len(title) > 0:
name_base = title
keywords = title.split()
else:
name_base, __ = os.path.splitext(
form.cleaned_data["base_file"].name)
keywords = name_base.split()
ignore_keys = KeywordIgnoreListModel.objects.values_list('key',
flat=True)
keyword_list = []
for key in keywords:
if key not in ignore_keys and not key.isdigit() and any(
c.isalpha() for c in key) and len(key) > 2:
keyword_list.append(key)
keywords = keyword_list
name = slugify(name_base.replace(".", "_"))
try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
# This should be followed up in upstream Django.
tempdir, base_file, file_type = form.write_files()
saved_layer = file_upload(
base_file,
name=name,
user=request.user,
category=category,
group=group,
keywords=keywords,
overwrite=False,
charset=form.cleaned_data["charset"],
abstract=form.cleaned_data["abstract"],
title=form.cleaned_data["layer_title"],
metadata_uploaded_preserve=form.
cleaned_data["metadata_uploaded_preserve"])
file = form.cleaned_data['base_file']
saved_layer.file_size = file.size
saved_layer.file_type = file_type
saved_layer.save()
if admin_upload:
saved_layer.status = 'ACTIVE'
saved_layer.save()
except Exception as e:
exception_type, error, tb = sys.exc_info()
logger.exception(e)
out['success'] = False
out['errors'] = str(error)
# Assign the error message to the latest UploadSession from that user.
latest_uploads = UploadSession.objects.filter(
user=request.user).order_by('-date')
if latest_uploads.count() > 0:
upload_session = latest_uploads[0]
upload_session.error = str(error)
upload_session.traceback = traceback.format_exc(tb)
upload_session.context = log_snippet(CONTEXT_LOG_FILE)
upload_session.save()
out['traceback'] = upload_session.traceback
out['context'] = upload_session.context
out['upload_session'] = upload_session.id
else:
out['success'] = True
if hasattr(saved_layer, 'info'):
out['info'] = saved_layer.info
out['url'] = reverse('layer_detail',
args=[saved_layer.service_typename])
upload_session = saved_layer.upload_session
upload_session.processed = True
upload_session.save()
permissions = form.cleaned_data["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
permissions = saved_layer.resolvePermission(permissions)
saved_layer.set_permissions(permissions)
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
else:
for e in form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
if out['success']:
status_code = 200
else:
status_code = 400
return HttpResponse(json.dumps(out),
content_type='application/json',
status=status_code)