def test_layer_delete_from_geonetwork(self):
"""Verify that layer is correctly deleted from GeoNetwork
"""
gn_cat = Layer.objects.gn_catalog
# Test Uploading then Deleting a Shapefile from GeoNetwork
shp_file = os.path.join(TEST_DATA, "lembang_schools.shp")
shp_layer = file_upload(shp_file)
shp_layer.delete_from_geonetwork()
shp_layer_info = gn_cat.get_by_uuid(shp_layer.uuid)
assert shp_layer_info == None
# Clean up and completely delete the layer
shp_layer.delete()
# Test Uploading then Deleting a TIFF file from GeoNetwork
tif_file = os.path.join(TEST_DATA, "test_grid.tif")
tif_layer = file_upload(tif_file)
tif_layer.delete_from_geonetwork()
tif_layer_info = gn_cat.get_by_uuid(tif_layer.uuid)
assert tif_layer_info == None
# Clean up and completely delete the layer
tif_layer.delete()
def test_layer_delete_from_geoserver(self):
"""Verify that layer is correctly deleted from GeoServer
"""
# Layer.delete_from_geoserver() uses cascading_delete()
# Should we explicitly test that the styles and store are
# deleted as well as the resource itself?
# There is already an explicit test for cascading delete
gs_cat = Layer.objects.gs_catalog
# Test Uploading then Deleting a Shapefile from GeoServer
shp_file = os.path.join(TEST_DATA, "lembang_schools.shp")
shp_layer = file_upload(shp_file)
shp_store = gs_cat.get_store(shp_layer.name)
shp_layer.delete_from_geoserver()
self.assertRaises(FailedRequestError, lambda: gs_cat.get_resource(shp_layer.name, store=shp_store))
shp_layer.delete()
# Test Uploading then Deleting a TIFF file from GeoServer
tif_file = os.path.join(TEST_DATA, "test_grid.tif")
tif_layer = file_upload(tif_file)
tif_store = gs_cat.get_store(tif_layer.name)
tif_layer.delete_from_geoserver()
self.assertRaises(FailedRequestError, lambda: gs_cat.get_resource(shp_layer.name, store=tif_store))
tif_layer.delete()
def test_extension_not_implemented(self):
"""Verify a GeoNodeException is returned for not compatible extensions
"""
sampletxt = os.path.join(TEST_DATA, "lembang_schools_percentage_loss.dbf")
try:
file_upload(sampletxt)
except GeoNodeException, e:
pass
def test_cascading_delete(self):
"""Verify that the gs_helpers.cascading_delete() method is working properly
"""
gs_cat = Layer.objects.gs_catalog
# Upload a Shapefile
shp_file = os.path.join(TEST_DATA, "lembang_schools.shp")
shp_layer = file_upload(shp_file)
# Save the names of the Resource/Store/Styles
resource_name = shp_layer.resource.name
store = shp_layer.resource.store
store_name = store.name
layer = gs_cat.get_layer(resource_name)
styles = layer.styles + [layer.default_style]
# Delete the Layer using cascading_delete()
cascading_delete(gs_cat, shp_layer.resource)
# Verify that the styles were deleted
for style in styles:
s = gs_cat.get_style(style)
assert s == None
# Verify that the resource was deleted
self.assertRaises(FailedRequestError, lambda: gs_cat.get_resource(resource_name, store=store))
# Verify that the store was deleted
self.assertRaises(FailedRequestError, lambda: gs_cat.get_store(store_name))
# Clean up by deleting the layer from GeoNode's DB and GeoNetwork
shp_layer.delete()
def test_delete_layer(self):
"""Verify that the 'delete_layer' pre_delete hook is functioning
"""
gs_cat = Layer.objects.gs_catalog
gn_cat = Layer.objects.gn_catalog
# Upload a Shapefile Layer
shp_file = os.path.join(TEST_DATA, "lembang_schools.shp")
shp_layer = file_upload(shp_file)
shp_layer_id = shp_layer.pk
shp_store = gs_cat.get_store(shp_layer.name)
shp_store_name = shp_store.name
id = shp_layer.pk
name = shp_layer.name
uuid = shp_layer.uuid
# Delete it with the Layer.delete() method
shp_layer.delete()
# Verify that it no longer exists in GeoServer
self.assertRaises(FailedRequestError, lambda: gs_cat.get_resource(name, store=shp_store))
self.assertRaises(FailedRequestError, lambda: gs_cat.get_store(shp_store_name))
# Verify that it no longer exists in GeoNetwork
shp_layer_gn_info = gn_cat.get_by_uuid(uuid)
assert shp_layer_gn_info == None
# Check that it was also deleted from GeoNodes DB
self.assertRaises(ObjectDoesNotExist, lambda: Layer.objects.get(pk=shp_layer_id))
def test_file_upload(self):
"""
Tests that a file can be uploaded without associating it with an specific event.
"""
thefile = os.path.join(TEST_DATA, 'TM_WORLD.shp')
uploaded = file_upload(thefile, overwrite=True)
assert uploaded.typename is not None
def test_empty_bbox(self):
"""Regression-test for failures caused by zero-width bounding boxes"""
thefile = os.path.join(TEST_DATA, "single_point.shp")
uploaded = file_upload(thefile, overwrite=True)
detail_page_url = urljoin(settings.SITEURL, uploaded.get_absolute_url())
get_web_page(detail_page_url) # no assertion, but this will fail if the page 500's
new_map_url = urljoin(settings.SITEURL, "/maps/new") + "?" + urlencode(dict(layer=uploaded.typename))
get_web_page(new_map_url) # no assertion, but this will fail if the page 500's
def test_event_file_upload(self):
"""
Tests that a file can be uploaded and associated with an specific event.
"""
thefile = os.path.join(TEST_DATA, 'KEN_Affected_Regions.shp')
uploaded = file_upload(thefile, workspace=self.event.slug, overwrite=True)
self.assertEqual(self.event.slug, uploaded.workspace)
def test_tiff(self):
"""Uploading a good .tiff
"""
thefile = os.path.join(TEST_DATA, "test_grid.tif")
uploaded = file_upload(thefile)
check_layer(uploaded)
# Clean up and completely delete the layer
uploaded.delete()
def test_bad_shapefile(self):
"""Verifying GeoNode complains about a shapefile without .prj
"""
thefile = os.path.join(TEST_DATA, "lembang_schools_percentage_loss.shp")
try:
uploaded = file_upload(thefile)
except GeoNodeException, e:
pass
def test_shapefile(self):
"""Test Uploading a good shapefile
"""
thefile = os.path.join(TEST_DATA, "lembang_schools.shp")
uploaded = file_upload(thefile)
check_layer(uploaded)
# Clean up and completely delete the layer
uploaded.delete()
def test_keywords_upload(self):
"""Check that keywords can be passed to file_upload
"""
thefile = os.path.join(TEST_DATA, "lembang_schools.shp")
uploaded = file_upload(thefile, keywords=["foo", "bar"], overwrite=True)
keywords = uploaded.keyword_list()
msg = "No keywords found in layer %s" % uploaded.name
assert len(keywords) > 0, msg
assert "foo" in uploaded.keyword_list(), 'Could not find "foo" in %s' % keywords
assert "bar" in uploaded.keyword_list(), 'Could not find "bar" in %s' % keywords
def test_repeated_upload(self):
"""Upload the same file more than once
"""
thefile = os.path.join(TEST_DATA, "test_grid.tif")
uploaded1 = file_upload(thefile)
check_layer(uploaded1)
uploaded2 = file_upload(thefile, overwrite=True)
check_layer(uploaded2)
uploaded3 = file_upload(thefile, overwrite=False)
check_layer(uploaded3)
msg = "Expected %s but got %s" % (uploaded1.name, uploaded2.name)
assert uploaded1.name == uploaded2.name, msg
msg = "Expected a different name when uploading %s using " "overwrite=False but got %s" % (
thefile,
uploaded3.name,
)
assert uploaded1.name != uploaded3.name, msg
# Clean up and completely delete the layers
# uploaded1 is overwritten by uploaded2 ... no need to delete it
uploaded2.delete()
uploaded3.delete()
def save_file_to_geonode(filename, user=None, title=None,
overwrite=True, check_metadata=True,
ignore=None):
"""Save a single layer file to local Risiko GeoNode
Input
filename: Layer filename of type as defined in LAYER_TYPES
user: Django User object
title: String describing the layer.
If None or '' the filename will be used.
overwrite: Boolean variable controlling whether existing layers
can be overwritten by this operation. Default is True
check_metadata: Flag controlling whether metadata is verified.
If True (default), an exception will be raised
if metada is not available after a number of retries.
If False, no check is done making the function faster.
Output
layer object
"""
if ignore is not None and filename == ignore:
return None
# Extract fully qualified basename and extension
basename, extension = os.path.splitext(filename)
if extension not in LAYER_TYPES:
msg = ('Invalid file extension in file %s. Valid extensions are '
'%s' % (filename, str(LAYER_TYPES)))
raise RisikoException(msg)
# Use file name to derive title if not specified
if title is None or title == '':
title = os.path.split(basename)[-1]
# Try to find a file with a .keywords extension
# and create a keywords list from there.
# It is assumed that the keywords are separated
# by new lines.
# Empty keyword lines are ignored (as this causes issues downstream)
keyword_list = []
keyword_file = basename + '.keywords'
if os.path.exists(keyword_file):
f = open(keyword_file, 'r')
for line in f.readlines():
# Ignore blank lines
raw_keyword = line.strip()
if raw_keyword == '':
continue
# Strip any spaces after or before the colons if present
if ':' in raw_keyword:
keyword = ':'.join([x.strip() for x in raw_keyword.split(':')])
# Store keyword
keyword_list.append(keyword)
f.close()
# Take care of file types
if extension == '.asc':
# We assume this is an AAIGrid ASCII file such as those generated by
# ESRI and convert it to Geotiff before uploading.
# Create temporary tif file for upload and check that the road is clear
prefix = os.path.split(basename)[-1]
upload_filename = unique_filename(prefix=prefix, suffix='.tif')
upload_basename, extension = os.path.splitext(upload_filename)
# Copy any metadata files to unique filename
for ext in ['.sld', '.keywords']:
if os.path.exists(basename + ext):
cmd = 'cp %s%s %s%s' % (basename, ext, upload_basename, ext)
run(cmd)
# Check that projection file exists
prjname = basename + '.prj'
if not os.path.isfile(prjname):
msg = ('File %s must have a projection file named '
'%s' % (filename, prjname))
raise RisikoException(msg)
# Convert ASCII file to GeoTIFF
R = read_layer(filename)
R.write_to_file(upload_filename)
else:
# The specified file is the one to upload
upload_filename = filename
# Attempt to upload the layer
try:
# Upload
layer = file_upload(upload_filename,
user=user,
title=title,
keywords=keyword_list,
overwrite=overwrite)
# FIXME (Ole): This is some kind of hack that should be revisited.
layer.keywords = ' '.join(keyword_list)
layer.save()
except GeoNodeException, e:
# Layer did not upload. Convert GeoNodeException to RisikoException
raise RisikoException(e)
def save_file_to_geonode(filename,
user=None,
title=None,
overwrite=True,
check_metadata=True,
ignore=None):
"""Save a single layer file to local Risiko GeoNode
Input
filename: Layer filename of type as defined in LAYER_TYPES
user: Django User object
title: String describing the layer.
If None or '' the filename will be used.
overwrite: Boolean variable controlling whether existing layers
can be overwritten by this operation. Default is True
check_metadata: Flag controlling whether metadata is verified.
If True (default), an exception will be raised
if metada is not available after a number of retries.
If False, no check is done making the function faster.
Output
layer object
"""
if ignore is not None and filename == ignore:
return None
# Extract fully qualified basename and extension
basename, extension = os.path.splitext(filename)
if extension not in LAYER_TYPES:
msg = ('Invalid file extension in file %s. Valid extensions are '
'%s' % (filename, str(LAYER_TYPES)))
raise RisikoException(msg)
# Use file name to derive title if not specified
if title is None or title == '':
title = os.path.split(basename)[-1]
# Try to find a file with a .keywords extension
# and create a keywords list from there.
# It is assumed that the keywords are separated
# by new lines.
# Empty keyword lines are ignored (as this causes issues downstream)
keyword_list = []
keyword_file = basename + '.keywords'
if os.path.exists(keyword_file):
f = open(keyword_file, 'r')
for line in f.readlines():
# Ignore blank lines
raw_keyword = line.strip()
if raw_keyword == '':
continue
# Strip any spaces after or before the colons if present
if ':' in raw_keyword:
keyword = ':'.join([x.strip() for x in raw_keyword.split(':')])
# FIXME (Ole): Replace spaces by underscores and store keyword.
# See issue #148
keyword_list.append(keyword.replace(' ', '_'))
f.close()
# Take care of file types
if extension == '.asc':
# We assume this is an AAIGrid ASCII file such as those generated by
# ESRI and convert it to Geotiff before uploading.
# Create temporary tif file for upload and check that the road is clear
prefix = os.path.split(basename)[-1]
upload_filename = unique_filename(prefix=prefix, suffix='.tif')
upload_basename, extension = os.path.splitext(upload_filename)
# Copy any metadata files to unique filename
for ext in ['.sld', '.keywords']:
if os.path.exists(basename + ext):
cmd = 'cp %s%s %s%s' % (basename, ext, upload_basename, ext)
run(cmd)
# Check that projection file exists
prjname = basename + '.prj'
if not os.path.isfile(prjname):
msg = ('File %s must have a projection file named '
'%s' % (filename, prjname))
raise RisikoException(msg)
# Convert ASCII file to GeoTIFF
R = read_layer(filename)
R.write_to_file(upload_filename)
else:
# The specified file is the one to upload
upload_filename = filename
# Attempt to upload the layer
try:
# Upload
layer = file_upload(upload_filename,
user=user,
title=title,
keywords=keyword_list,
overwrite=overwrite)
# FIXME (Ole): This workaround should be revisited.
# This fx means that keywords can't have spaces
# Really need a generic way of getting this kind of
# info in and out of GeoNode
layer.keywords = ' '.join(keyword_list)
layer.save()
except GeoNodeException, e:
# Layer did not upload. Convert GeoNodeException to RisikoException
raise RisikoException(e)
def save_file_to_geonode(filename, user=None, title=None,
overwrite=False):
"""Save a single layer file to local Risiko GeoNode
Input
filename: Layer filename of type as defined in LAYER_TYPES
user: Django User object
title: String describing the layer.
If None or '' the filename will be used.
overwrite: Boolean variable controlling whether existing layers
can be overwritten by this operation. Default is False
Output
layer object
"""
# Extract fully qualified basename and extension
basename, extension = os.path.splitext(filename)
if extension not in LAYER_TYPES:
msg = ('Invalid file extension in file %s. Valid extensions are '
'%s' % (filename, str(LAYER_TYPES)))
raise RisikoException(msg)
# Use file name to derive title if not specified
if title is None or title == '':
title = os.path.split(basename)[-1]
# Try to find a file with a .keywords extension
# and create a keywords list from there.
# It is assumed that the keywords are separated
# by new lines.
keyword_list = []
keyword_file = basename + '.keywords'
if os.path.exists(keyword_file):
f = open(keyword_file, 'r')
for line in f.readlines():
# Strip any spaces after or before the colons if present
raw_keyword = line.strip()
if ':' in raw_keyword:
keyword = ':'.join([x.strip() for x in raw_keyword.split(':')])
keyword_list.append(keyword)
f.close()
# Take care of file types
if extension == '.asc':
# We assume this is an AAIGrid ASCII file such as those generated by
# ESRI and convert it to Geotiff before uploading.
# Create temporary tif file for upload and check that the road is clear
# FIXME (Ole): for some reason, these files tend to hang around
# - especially after interrupts so we'll go for temporary filenames
# for the time being.
prefix=os.path.split(basename)[-1]
upload_filename = unique_filename(prefix=prefix, suffix='.tif')
upload_basename, extension = os.path.splitext(upload_filename)
# Copy any metadata files to unique filename
for ext in ['.sld', '.keywords']:
if os.path.exists(basename+ext):
cmd = 'cp %s%s %s%s' % (basename, ext, upload_basename, ext)
run(cmd)
#msg = ('You have asked to upload the ASCII file "%s" and to do so I '
# 'must first convert it to the TIF format. However, there is '
# 'already a file named "%s" so you have to remove that first '
# 'and try again. Sorry about that.' % (filename,
# upload_filename))
#assert not os.path.exists(upload_filename), msg
# Check that projection file exists
prjname = basename + '.prj'
if not os.path.isfile(prjname):
msg = ('File %s must have a projection file named '
'%s' % (filename, prjname))
raise RisikoException(msg)
# Convert ASCII file to GeoTIFF
cmd = ('gdal_translate -ot Float64 -of GTiff -co "PROFILE=GEOTIFF" '
'%s %s' % (filename, upload_filename))
run(cmd,
stdout='%s_asc2tif_conversion.stdout' % basename,
stderr='%s_asc2tif_conversion.stderr' % basename)
else:
# The specified file is the one to upload
upload_filename = filename
# Attempt to upload the layer
try:
# Upload
layer = file_upload(upload_filename,
user=user,
title=title,
keywords=keyword_list,
overwrite=overwrite)
except GeoNodeException, e:
# Layer did not upload. Convert GeoNodeException to RisikoException
raise RisikoException(e)
