def test_metadata_twice(self):
"""Layer metadata can be correctly uploaded multiple times
"""
# This test reproduces ticket #99 by creating new data,
# uploading twice and verifying metadata
# Base test data
filenames = ['jakarta_flood_design.tif', ]
for org_filename in filenames:
org_basename, ext = os.path.splitext(os.path.join(UNITDATA, 'hazard',
org_filename))
# Copy data to temporary unique name
basename = unique_filename(dir='/tmp')
cmd = '/bin/cp -f %s.keywords %s.keywords' % (org_basename, basename)
os.system(cmd)
# Not needed since we are dealing with a raster
#cmd = '/bin/cp -f %s.prj %s.prj' % (org_basename, basename)
#os.system(cmd)
if ext == '.tif':
layertype = 'raster'
filename = '%s.tif' % basename
cmd = '/bin/cp %s.tif %s' % (org_basename, filename)
os.system(cmd)
elif ext == '.shp':
layertype = 'vector'
filename = '%s.shp' % basename
for e in ['shp', 'shx', 'sbx', 'sbn', 'dbf']:
cmd = '/bin/cp %s.%s %s.%s' % (org_basename, e,
basename, e)
os.system(cmd)
else:
msg = ('Unknown layer extension in %s. '
'Expected .shp or .asc' % filename)
raise Exception(msg)
# Repeat multiple times
for i in range(3):
# Upload
layer = save_to_geonode(filename, user=self.user,
overwrite=True)
# Get metadata
layer_name = '%s:%s' % (layer.workspace, layer.name)
metadata = get_metadata(INTERNAL_SERVER_URL,
layer_name)
# Verify
assert 'id' in metadata
assert 'title' in metadata
assert 'layertype' in metadata
assert 'keywords' in metadata
assert 'bounding_box' in metadata
assert len(metadata['bounding_box']) == 4
# Check integrity between Django layer and file
assert_bounding_box_matches(layer, filename)
# Check integrity between file and OWS metadata
ref_bbox = get_bounding_box(filename)
msg = ('Bounding box from OWS did not match bounding box '
'from file. They are\n'
'From file %s: %s\n'
'From OWS: %s' % (filename,
ref_bbox,
metadata['bounding_box']))
assert numpy.allclose(metadata['bounding_box'],
ref_bbox), msg
assert layer.title == metadata['title']
assert layer_name == metadata['id']
assert layertype == metadata['layertype']
# Check keywords
if layertype == 'raster':
category = 'hazard'
subcategory = 'flood'
else:
msg = 'Unknown layer type %s' % layertype
raise Exception(msg)
keywords = metadata['keywords']
msg = 'Did not find key "category" in keywords: %s' % keywords
assert 'category' in keywords, msg
msg = ('Did not find key "subcategory" in keywords: %s'
% keywords)
assert 'subcategory' in keywords, msg
msg = ('Category keyword %s did not match expected %s'
% (keywords['category'], category))
assert category == keywords['category'], msg
msg = ('Subcategory keyword %s did not match expected %s'
% (keywords['subcategory'], category))
assert subcategory == keywords['subcategory'], msg
def save_file_to_geonode(filename, user=None, title=None,
overwrite=True, check_metadata=True,
ignore=None):
"""Save a single layer file to local Risiko GeoNode
Input
filename: Layer filename of type as defined in LAYER_TYPES
user: Django User object
title: String describing the layer.
If None or '' the filename will be used.
overwrite: Boolean variable controlling whether existing layers
can be overwritten by this operation. Default is True
check_metadata: Flag controlling whether metadata is verified.
If True (default), an exception will be raised
if metada is not available after a number of retries.
If False, no check is done making the function faster.
Output
layer object
"""
if ignore is not None and filename == ignore:
return None
# Extract fully qualified basename and extension
basename, extension = os.path.splitext(filename)
if extension not in LAYER_TYPES:
msg = ('Invalid file extension "%s" in file %s. Valid extensions are '
'%s' % (extension, filename, str(LAYER_TYPES)))
raise RisikoException(msg)
# Try to find a file with a .keywords extension
# and create a keywords list from there.
# It is assumed that the keywords are separated
# by new lines.
# Empty keyword lines are ignored (as this causes issues downstream)
keyword_list = []
keyword_file = basename + '.keywords'
kw_title = title if title is not None else None
kw_summary = None
kw_table = None
if os.path.exists(keyword_file):
f = open(keyword_file, 'r')
for line in f.readlines():
# Ignore blank lines
raw_keyword = line.strip()
if raw_keyword == '':
continue
# Strip any spaces after or before the colons if present
if ':' in raw_keyword:
keyword = ':'.join([x.strip() for x in raw_keyword.split(':')])
# Grab title if present
if 'title' in keyword and kw_title is None:
kw_title = keyword.split(':')[1]
if 'impact_summary' in keyword:
kw_summary = ''.join(keyword.split(':')[1:])
continue
if 'impact_table' in keyword:
kw_table = keyword.split(':')[1]
continue
keyword_list.append(keyword)
f.close()
# Take care of file types
if extension == '.asc':
# We assume this is an AAIGrid ASCII file such as those generated by
# ESRI and convert it to Geotiff before uploading.
# Create temporary tif file for upload and check that the road is clear
prefix = os.path.split(basename)[-1]
upload_filename = unique_filename(prefix=prefix, suffix='.tif')
upload_basename, extension = os.path.splitext(upload_filename)
# Copy any metadata files to unique filename
for ext in ['.sld', '.keywords']:
if os.path.exists(basename + ext):
cmd = 'cp %s%s %s%s' % (basename, ext, upload_basename, ext)
run(cmd)
# Check that projection file exists
prjname = basename + '.prj'
if not os.path.isfile(prjname):
msg = ('File %s must have a projection file named '
'%s' % (filename, prjname))
raise RisikoException(msg)
# Convert ASCII file to GeoTIFF
R = read_layer(filename)
R.write_to_file(upload_filename)
else:
# The specified file is the one to upload
upload_filename = filename
# Use file name or keywords to derive title if not specified
if kw_title is None:
title = os.path.split(basename)[-1]
else:
title = kw_title
# Attempt to upload the layer
try:
# Upload
layer = file_upload(upload_filename,
user=user,
title=title,
keywords=keyword_list,
overwrite=overwrite)
if kw_summary is not None:
layer.abstract = kw_summary
if kw_table is not None:
layer.supplemental_information = kw_table
if kw_title is not None:
layer.title = kw_title
layer.save()
except GeoNodeException, e:
raise
def test_metadata_twice(self):
"""Layer metadata can be correctly uploaded multiple times
"""
# This test reproduces ticket #99 by creating new data,
# uploading twice and verifying metadata
# Base test data
filenames = [
'jakarta_flood_design.tif',
]
for org_filename in filenames:
org_basename, ext = os.path.splitext(
os.path.join(UNITDATA, 'hazard', org_filename))
# Copy data to temporary unique name
basename = unique_filename(dir='/tmp')
cmd = '/bin/cp -f %s.keywords %s.keywords' % (org_basename,
basename)
os.system(cmd)
# Not needed since we are dealing with a raster
#cmd = '/bin/cp -f %s.prj %s.prj' % (org_basename, basename)
#os.system(cmd)
if ext == '.tif':
layertype = 'raster'
filename = '%s.tif' % basename
cmd = '/bin/cp %s.tif %s' % (org_basename, filename)
os.system(cmd)
elif ext == '.shp':
layertype = 'vector'
filename = '%s.shp' % basename
for e in ['shp', 'shx', 'sbx', 'sbn', 'dbf']:
cmd = '/bin/cp %s.%s %s.%s' % (org_basename, e, basename,
e)
os.system(cmd)
else:
msg = ('Unknown layer extension in %s. '
'Expected .shp or .asc' % filename)
raise Exception(msg)
# Repeat multiple times
for i in range(3):
# Upload
layer = save_to_geonode(filename,
user=self.user,
overwrite=True)
# Get metadata
layer_name = '%s:%s' % (layer.workspace, layer.name)
metadata = get_metadata(INTERNAL_SERVER_URL, layer_name)
# Verify
assert 'id' in metadata
assert 'title' in metadata
assert 'layertype' in metadata
assert 'keywords' in metadata
assert 'bounding_box' in metadata
assert len(metadata['bounding_box']) == 4
# Check integrity between Django layer and file
assert_bounding_box_matches(layer, filename)
# Check integrity between file and OWS metadata
ref_bbox = get_bounding_box(filename)
msg = ('Bounding box from OWS did not match bounding box '
'from file. They are\n'
'From file %s: %s\n'
'From OWS: %s' %
(filename, ref_bbox, metadata['bounding_box']))
assert numpy.allclose(metadata['bounding_box'], ref_bbox), msg
assert layer.title == metadata['title']
assert layer_name == metadata['id']
assert layertype == metadata['layertype']
# Check keywords
if layertype == 'raster':
category = 'hazard'
subcategory = 'flood'
else:
msg = 'Unknown layer type %s' % layertype
raise Exception(msg)
keywords = metadata['keywords']
msg = 'Did not find key "category" in keywords: %s' % keywords
assert 'category' in keywords, msg
msg = ('Did not find key "subcategory" in keywords: %s' %
keywords)
assert 'subcategory' in keywords, msg
msg = ('Category keyword %s did not match expected %s' %
(keywords['category'], category))
assert category == keywords['category'], msg
msg = ('Subcategory keyword %s did not match expected %s' %
(keywords['subcategory'], category))
assert subcategory == keywords['subcategory'], msg