def test_keywords_with_colon(self):
"""Keywords and values with colons raise error messages
"""
# Colon in key
kwd_filename = unique_filename(suffix='.keywords')
keywords = {'with_a_colon:in_it': 'value'} # This one is illegal
try:
write_keywords(keywords, kwd_filename)
except AssertionError:
pass
else:
msg = 'Colon in keywords key %s was not caught' % keywords
raise Exception(msg)
# Colon in value
kwd_filename = unique_filename(suffix='.keywords')
keywords = {'with_a_colon': 'take: that!'} # This one is illegal
try:
write_keywords(keywords, kwd_filename)
except AssertionError:
pass
else:
msg = 'Colon in keywords value %s was not caught' % keywords
raise Exception(msg)
def test_keywords_file(self):
"""Keywords can be written and read
"""
kwd_filename = unique_filename(suffix='.keywords')
keywords = {'caption': 'Describing the layer',
'category': 'impact',
'subcategory': 'flood',
'layer': None,
'with spaces': 'trailing_ws '}
write_keywords(keywords, kwd_filename)
msg = 'Keywords file %s was not created' % kwd_filename
assert os.path.isfile(kwd_filename), msg
x = read_keywords(kwd_filename)
os.remove(kwd_filename)
assert isinstance(x, dict)
# Check keyword names
for key in x:
msg = 'Read unexpected key %s' % key
assert key in keywords, msg
for key in keywords:
msg = 'Expected key %s was not read from %s' % (key,
kwd_filename)
assert key in x, msg
# Check keyword values
for key in keywords:
refval = keywords[key]
newval = x[key]
if refval is None:
assert newval is None
else:
msg = ('Expected value %s was not read from %s. '
'I got %s' % (refval, kwd_filename, newval))
assert refval.strip() == newval, msg
# Check catching of wrong extension
kwd_filename = unique_filename(suffix='.xxxx')
try:
write_keywords(keywords, kwd_filename)
except:
pass
else:
msg = 'Should have raised assertion error for wrong extension'
raise Exception(msg)
def test_empty_keywords_file(self):
"""Empty keywords can be handled
"""
kwd_filename = unique_filename(suffix='.keywords')
write_keywords({}, kwd_filename)
msg = 'Keywords file %s was not created' % kwd_filename
assert os.path.isfile(kwd_filename), msg
x = read_keywords(kwd_filename)
os.remove(kwd_filename)
assert isinstance(x, dict)
assert len(x) == 0
def write_to_file(self, filename):
"""Save raster data to file
Input
filename: filename with extension .tif
"""
# Check file format
basename, extension = os.path.splitext(filename)
msg = ('Invalid file type for file %s. Only extension '
'tif allowed.' % filename)
assert extension in ['.tif', '.asc'], msg
format = DRIVER_MAP[extension]
# Get raster data
A = self.get_data()
# Get Dimensions. Note numpy and Gdal swap order
N, M = A.shape
# Create empty file.
# FIXME (Ole): It appears that this is created as single
# precision even though Float64 is specified
# - see issue #17
driver = gdal.GetDriverByName(format)
fid = driver.Create(filename, M, N, 1, gdal.GDT_Float64)
if fid is None:
msg = ('Gdal could not create filename %s using '
'format %s' % (filename, format))
raise Exception(msg)
# Write metada
fid.SetProjection(str(self.projection))
fid.SetGeoTransform(self.geotransform)
# Write data
fid.GetRasterBand(1).WriteArray(A)
# Write keywords if any
write_keywords(self.keywords, basename + '.keywords')
def write_to_file(self, filename):
"""Save raster data to file
Input
filename: filename with extension .tif
"""
# Check file format
basename, extension = os.path.splitext(filename)
msg = "Invalid file type for file %s. Only extension " "tif allowed." % filename
assert extension in [".tif", ".asc"], msg
format = DRIVER_MAP[extension]
# Get raster data
A = self.get_data()
# Get Dimensions. Note numpy and Gdal swap order
N, M = A.shape
# Create empty file
driver = gdal.GetDriverByName(format)
fid = driver.Create(filename, M, N, 1, gdal.GDT_Float64)
if fid is None:
msg = "Gdal could not create filename %s using " "format %s" % (filename, format)
raise Exception(msg)
# Write metada
fid.SetProjection(str(self.projection))
fid.SetGeoTransform(self.geotransform)
# Write data
fid.GetRasterBand(1).WriteArray(A)
# Write keywords if any
write_keywords(self.keywords, basename + ".keywords")
def write_to_file(self, filename):
"""Save vector data to file
Input
filename: filename with extension .shp or .gml
Note, if attribute names are longer than 10 characters they will be
truncated. This is due to limitations in the shp file driver and has
to be done here since gdal v1.7 onwards has changed its handling of
this issue: http://www.gdal.org/ogr/drv_shapefile.html
"""
# Check file format
basename, extension = os.path.splitext(filename)
msg = "Invalid file type for file %s. Only extensions " "shp or gml allowed." % filename
assert extension == ".shp" or extension == ".gml", msg
driver = DRIVER_MAP[extension]
# FIXME (Ole): Tempory flagging of GML issue (ticket #18)
if extension == ".gml":
msg = (
"OGR GML driver does not store geospatial reference."
"This format is disabled for the time being. See "
"https://github.com/AIFDR/riab/issues/18"
)
raise Exception(msg)
# Derive layername from filename (excluding preceding dirs)
layername = os.path.split(basename)[-1]
# Get vector data
geometry = self.get_geometry()
data = self.get_data()
N = len(geometry)
# Clear any previous file of this name (ogr does not overwrite)
try:
os.remove(filename)
except:
pass
# Create new file with one layer
drv = ogr.GetDriverByName(driver)
if drv is None:
msg = "OGR driver %s not available" % driver
raise Exception(msg)
ds = drv.CreateDataSource(filename)
if ds is None:
msg = "Creation of output file %s failed" % filename
raise Exception(msg)
lyr = ds.CreateLayer(layername, self.projection.spatial_reference, self.geometry_type)
if lyr is None:
msg = "Could not create layer %s" % layername
raise Exception(msg)
# Define attributes if any
store_attributes = False
if data is not None:
if len(data) > 0:
try:
fields = data[0].keys()
except:
msg = (
'Input parameter "attributes" was specified '
"but it does not contain dictionaries with "
"field information as expected. The first"
"element is %s" % data[0]
)
raise Exception(msg)
else:
# Establish OGR types for each element
ogrtypes = {}
for name in fields:
att = data[0][name]
py_type = type(att)
msg = "Unknown type for storing vector " "data: %s, %s" % (name, str(py_type)[1:-1])
assert py_type in TYPE_MAP, msg
ogrtypes[name] = TYPE_MAP[py_type]
else:
msg = 'Input parameter "data" was specified ' "but appears to be empty"
raise Exception(msg)
# Create attribute fields in layer
store_attributes = True
for name in fields:
fd = ogr.FieldDefn(name, ogrtypes[name])
# FIXME (Ole): Trying to address issue #16
# But it doesn't work and
# somehow changes the values of MMI in test
# width = max(128, len(name))
# print name, width
# fd.SetWidth(width)
# Silent handling of warnings like
# Warning 6: Normalized/laundered field name:
#'CONTENTS_LOSS_AUD' to 'CONTENTS_L'
gdal.PushErrorHandler("CPLQuietErrorHandler")
if lyr.CreateField(fd) != 0:
msg = "Could not create field %s" % name
raise Exception(msg)
# Restore error handler
gdal.PopErrorHandler()
# Store geometry
geom = ogr.Geometry(self.geometry_type)
layer_def = lyr.GetLayerDefn()
for i in range(N):
# Create new feature instance
feature = ogr.Feature(layer_def)
# Store geometry and check
if self.geometry_type == ogr.wkbPoint:
x = float(geometry[i][0])
y = float(geometry[i][1])
geom.SetPoint_2D(0, x, y)
elif self.geometry_type == ogr.wkbPolygon:
wkt = array2wkt(geometry[i], geom_type="POLYGON")
geom = ogr.CreateGeometryFromWkt(wkt)
else:
msg = "Geometry type %s not implemented" % self.geometry_type
raise Exception(msg)
feature.SetGeometry(geom)
G = feature.GetGeometryRef()
if G is None:
msg = "Could not create GeometryRef for file %s" % filename
raise Exception(msg)
# Store attributes
if store_attributes:
for j, name in enumerate(fields):
actual_field_name = layer_def.GetFieldDefn(j).GetNameRef()
val = data[i][name]
if type(val) == numpy.ndarray:
# A singleton of type <type 'numpy.ndarray'> works
# for gdal version 1.6 but fails for version 1.8
# in SetField with error: NotImplementedError:
# Wrong number of arguments for overloaded function
val = float(val)
feature.SetField(actual_field_name, val)
# Save this feature
if lyr.CreateFeature(feature) != 0:
msg = "Failed to create feature %i in file %s" % (i, filename)
raise Exception(msg)
feature.Destroy()
# Write keywords if any
write_keywords(self.keywords, basename + ".keywords")
if resolution is None:
# Get native resolution and use that
resolution = layer_metadata['resolution']
#resolution = (resolution, resolution) #FIXME (Ole): Make nicer
# Download raster using specified bounding box and resolution
template = WCS_TEMPLATE
suffix = '.tif'
download_url = template % (server_url, layer_name, bbox_string,
resolution[0], resolution[1])
filename = get_file(download_url, suffix)
# Write keywords file
keywords = layer_metadata['keywords']
write_keywords(keywords, os.path.splitext(filename)[0] + '.keywords')
# Instantiate layer from file
lyr = read_layer(filename)
# FIXME (Ariel) Don't monkeypatch the layer object
lyr.metadata = layer_metadata
return lyr
def dummy_save(filename, title, user, metadata=''):
"""Take a file-like object and uploads it to a GeoNode
"""
return 'http://dummy/data/geonode:' + filename + '_by_' + user.username
def write_to_file(self, filename):
"""Save vector data to file
Input
filename: filename with extension .shp or .gml
Note, if attribute names are longer than 10 characters they will be
truncated. This is due to limitations in the shp file driver and has
to be done here since gdal v1.7 onwards has changed its handling of
this issue: http://www.gdal.org/ogr/drv_shapefile.html
"""
# Check file format
basename, extension = os.path.splitext(filename)
msg = ('Invalid file type for file %s. Only extensions '
'shp or gml allowed.' % filename)
assert extension == '.shp' or extension == '.gml', msg
driver = DRIVER_MAP[extension]
# FIXME (Ole): Tempory flagging of GML issue (ticket #18)
if extension == '.gml':
msg = ('OGR GML driver does not store geospatial reference.'
'This format is disabled for the time being. See '
'https://github.com/AIFDR/riab/issues/18')
raise Exception(msg)
# Derive layername from filename (excluding preceding dirs)
layername = os.path.split(basename)[-1]
# Get vector data
geometry = self.get_geometry()
data = self.get_data()
N = len(geometry)
# Clear any previous file of this name (ogr does not overwrite)
try:
os.remove(filename)
except:
pass
# Create new file with one layer
drv = ogr.GetDriverByName(driver)
if drv is None:
msg = 'OGR driver %s not available' % driver
raise Exception(msg)
ds = drv.CreateDataSource(filename)
if ds is None:
msg = 'Creation of output file %s failed' % filename
raise Exception(msg)
lyr = ds.CreateLayer(layername, self.projection.spatial_reference,
self.geometry_type)
if lyr is None:
msg = 'Could not create layer %s' % layername
raise Exception(msg)
# Define attributes if any
store_attributes = False
if data is not None:
if len(data) > 0:
try:
fields = data[0].keys()
except:
msg = ('Input parameter "attributes" was specified '
'but it does not contain dictionaries with '
'field information as expected. The first'
'element is %s' % data[0])
raise Exception(msg)
else:
# Establish OGR types for each element
ogrtypes = {}
for name in fields:
att = data[0][name]
py_type = type(att)
msg = ('Unknown type for storing vector '
'data: %s, %s' % (name, str(py_type)[1:-1]))
assert py_type in TYPE_MAP, msg
ogrtypes[name] = TYPE_MAP[py_type]
else:
msg = ('Input parameter "data" was specified '
'but appears to be empty')
raise Exception(msg)
# Create attribute fields in layer
store_attributes = True
for name in fields:
fd = ogr.FieldDefn(name, ogrtypes[name])
# FIXME (Ole): Trying to address issue #16
# But it doesn't work and
# somehow changes the values of MMI in test
#width = max(128, len(name))
#print name, width
#fd.SetWidth(width)
# Silent handling of warnings like
# Warning 6: Normalized/laundered field name:
#'CONTENTS_LOSS_AUD' to 'CONTENTS_L'
gdal.PushErrorHandler('CPLQuietErrorHandler')
if lyr.CreateField(fd) != 0:
msg = 'Could not create field %s' % name
raise Exception(msg)
# Restore error handler
gdal.PopErrorHandler()
# Store geometry
geom = ogr.Geometry(self.geometry_type)
layer_def = lyr.GetLayerDefn()
for i in range(N):
# Create new feature instance
feature = ogr.Feature(layer_def)
# Store geometry and check
if self.geometry_type == ogr.wkbPoint:
x = float(geometry[i][0])
y = float(geometry[i][1])
geom.SetPoint_2D(0, x, y)
elif self.geometry_type == ogr.wkbPolygon:
wkt = array2wkt(geometry[i], geom_type='POLYGON')
geom = ogr.CreateGeometryFromWkt(wkt)
else:
msg = 'Geometry type %s not implemented' % self.geometry_type
raise Exception(msg)
feature.SetGeometry(geom)
G = feature.GetGeometryRef()
if G is None:
msg = 'Could not create GeometryRef for file %s' % filename
raise Exception(msg)
# Store attributes
if store_attributes:
for j, name in enumerate(fields):
actual_field_name = layer_def.GetFieldDefn(j).GetNameRef()
val = data[i][name]
if type(val) == numpy.ndarray:
# A singleton of type <type 'numpy.ndarray'> works
# for gdal version 1.6 but fails for version 1.8
# in SetField with error: NotImplementedError:
# Wrong number of arguments for overloaded function
val = float(val)
feature.SetField(actual_field_name, val)
# Save this feature
if lyr.CreateFeature(feature) != 0:
msg = 'Failed to create feature %i in file %s' % (i, filename)
raise Exception(msg)
feature.Destroy()
# Write keywords if any
write_keywords(self.keywords, basename + '.keywords')
def write_to_file(self, filename):
"""Save vector data to file
Input
filename: filename with extension .shp or .gml
"""
# Check file format
basename, extension = os.path.splitext(filename)
msg = ('Invalid file type for file %s. Only extensions '
'shp or gml allowed.' % filename)
assert extension == '.shp' or extension == '.gml', msg
driver = DRIVER_MAP[extension]
# FIXME (Ole): Tempory flagging of GML issue (ticket #18)
if extension == '.gml':
msg = ('OGR GML driver does not store geospatial reference.'
'This format is disabled for the time being. See '
'https://github.com/AIFDR/riab/issues/18')
raise Exception(msg)
# Derive layername from filename (excluding preceding dirs)
layername = os.path.split(basename)[-1]
# Get vector data
geometry = self.get_geometry()
data = self.get_data()
N = len(geometry)
# Clear any previous file of this name (ogr does not overwrite)
try:
os.remove(filename)
except:
pass
# Create new file with one layer
drv = ogr.GetDriverByName(driver)
if drv is None:
msg = 'OGR driver %s not available' % driver
raise Exception(msg)
ds = drv.CreateDataSource(filename)
if ds is None:
msg = 'Creation of output file %s failed' % filename
raise Exception(msg)
lyr = ds.CreateLayer(layername,
self.projection.spatial_reference,
self.geometry_type)
if lyr is None:
msg = 'Could not create layer %s' % layername
raise Exception(msg)
# Define attributes if any
store_attributes = False
if data is not None:
if len(data) > 0:
try:
fields = data[0].keys()
except:
msg = ('Input parameter "attributes" was specified '
'but it does not contain dictionaries with '
'field information as expected. The first'
'element is %s' % data[0])
raise Exception(msg)
else:
# Establish OGR types for each element
ogrtypes = {}
for name in fields:
att = data[0][name]
py_type = type(att)
msg = ('Unknown type for storing vector '
'data: %s, %s' % (name, str(py_type)[1:-1]))
assert py_type in TYPE_MAP, msg
ogrtypes[name] = TYPE_MAP[py_type]
else:
msg = ('Input parameter "data" was specified '
'but appears to be empty')
raise Exception(msg)
# Create attribute fields in layer
store_attributes = True
for name in fields:
fd = ogr.FieldDefn(name, ogrtypes[name])
# FIXME (Ole): Trying to address issue #16
# But it doesn't work and
# somehow changes the values of MMI in test
#width = max(128, len(name))
#print name, width
#fd.SetWidth(width)
if lyr.CreateField(fd) != 0:
msg = 'Could not create field %s' % name
raise Exception(msg)
# Store geometry
geom = ogr.Geometry(self.geometry_type)
layer_def = lyr.GetLayerDefn()
for i in range(N):
# Create new feature instance
feature = ogr.Feature(layer_def)
# Store geometry and check
if self.geometry_type == ogr.wkbPoint:
x = float(geometry[i][0])
y = float(geometry[i][1])
geom.SetPoint_2D(0, x, y)
elif self.geometry_type == ogr.wkbPolygon:
wkt = array2wkt(geometry[i])
geom = ogr.CreateGeometryFromWkt(wkt)
else:
msg = 'Geometry %s not implemented' % self.geometry_type
raise Exception(msg)
feature.SetGeometry(geom)
G = feature.GetGeometryRef()
if G is None:
msg = 'Could not create GeometryRef for file %s' % filename
raise Exception(msg)
# Store attributes
if store_attributes:
for name in fields:
feature.SetField(name, data[i][name])
# Save this feature
if lyr.CreateFeature(feature) != 0:
msg = 'Failed to create feature %i in file %s' % (i, filename)
raise Exception(msg)
feature.Destroy()
# Write keywords if any
write_keywords(self.keywords, basename + '.keywords')
if resolution is None:
# Get native resolution and use that
resolution = layer_metadata['resolution']
#resolution = (resolution, resolution) #FIXME (Ole): Make nicer
# Download raster using specified bounding box and resolution
template = WCS_TEMPLATE
suffix = '.tif'
download_url = template % (server_url, layer_name, bbox_string,
resolution[0], resolution[1])
filename = get_file(download_url, suffix)
# Write keywords file
keywords = layer_metadata['keywords']
write_keywords(keywords, os.path.splitext(filename)[0] + '.keywords')
# Instantiate layer from file
lyr = read_layer(filename)
# FIXME (Ariel) Don't monkeypatch the layer object
lyr.metadata = layer_metadata
return lyr
def dummy_save(filename, title, user, metadata=''):
"""Take a file-like object and uploads it to a GeoNode
"""
return 'http://dummy/data/geonode:' + filename + '_by_' + user.username