def getGeoServerBoundingBox(geoserver_layername):
cat = Catalog("{0}/rest/".format(geoserver_connection), geoserver_username,
geoserver_password)
ws = cat.get_workspace(workspace)
resource = cat.get_resource(geoserver_layername, workspace=ws)
bbox = resource.latlon_bbox[:4]
solr_geom = 'ENVELOPE({0},{1},{2},{3})'.format(bbox[0], bbox[1], bbox[3],
bbox[2])
return solr_geom
def layer_exists(layer_name, store, workspace):
_user, _password = ogc_server_settings.credentials
url = ogc_server_settings.rest
gs_catalog = Catalog(url, _user, _password)
try:
layer = gs_catalog.get_resource(layer_name, store=store,
workspace=workspace)
return layer is not None
except FailedRequestError:
return False
def _create_geoserver_geonode_layer(new_table, sld_type, title, the_user,
permissions):
# Create the Layer in GeoServer from the table
try:
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
settings.OGC_SERVER['default']['USER'],
settings.OGC_SERVER['default']['PASSWORD'])
ds = cat.get_store("uploaded") # name of store in WFP-Geonode
cat.publish_featuretype(new_table, ds, "EPSG:4326", srs="EPSG:4326")
# changing the layer's title in the title set by the user
resource = cat.get_resource(new_table, workspace="geonode")
resource.title = title
cat.save(resource)
except Exception as e:
msg = "Error creating GeoServer layer for %s: %s" % (new_table, str(e))
print msg
return None, msg
# Create the Layer in GeoNode from the GeoServer Layer
try:
link_to_sld = "{location}styles/{sld_type}".format(
**{
'location': settings.OGC_SERVER['default']['LOCATION'],
'sld_type': sld_type
})
r = requests.get(link_to_sld)
sld = r.text
sld = sld.replace(
"name_of_layer", new_table
) # "name_of_layer" is set in the predefined sld in geoserver (polygon_style, line_style, point_style)
cat.create_style(new_table, sld, overwrite=True)
style = cat.get_style(new_table)
layer = cat.get_layer(new_table)
layer.default_style = style
cat.save(layer)
gs_slurp(owner=the_user, filter=new_table)
from geonode.base.models import ResourceBase
layer = ResourceBase.objects.get(title=title)
geoserver_post_save(layer, ResourceBase)
# assign permissions for this layer
permissions_dict = json.loads(permissions) # needs to be dictionary
if permissions_dict is not None and len(permissions_dict.keys()) > 0:
layer.set_permissions(permissions_dict)
except Exception as e:
msg = "Error creating GeoNode layer for %s: %s" % (new_table, str(e))
return None, msg
def layer_exists(layer_name, store, workspace):
_user, _password = ogc_server_settings.credentials
url = ogc_server_settings.rest
gs_catalog = Catalog(url, _user, _password)
try:
layer = gs_catalog.get_resource(layer_name,
store=store,
workspace=workspace)
return layer is not None
except FailedRequestError:
return False
def geoserver_post_save(instance, sender, **kwargs):
"""Save keywords to GeoServer
The way keywords are implemented requires the layer
to be saved to the database before accessing them.
"""
url = ogc_server_settings.rest
try:
gs_catalog = Catalog(url, _user, _password)
gs_resource = gs_catalog.get_resource(instance.name)
except (FailedRequestError, EnvironmentError) as e:
msg = ('Could not connect to geoserver at "%s"'
'to save information for layer "%s"' % (
ogc_server_settings.LOCATION, instance.name.encode('utf-8'))
)
logger.warn(msg, e)
# If geoserver is not online, there is no need to continue
return
# If there is no resource returned it could mean one of two things:
# a) There is a synchronization problem in geoserver
# b) The unit tests are running and another geoserver is running in the
# background.
# For both cases it is sensible to stop processing the layer
if gs_resource is None:
logger.warn('Could not get geoserver resource for %s' % instance)
return
gs_resource.keywords = instance.keyword_list()
#gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings,"BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_resource)
bbox = gs_resource.latlon_bbox
dx = float(bbox[1]) - float(bbox[0])
dy = float(bbox[3]) - float(bbox[2])
dataAspect = 1 if dy == 0 else dx / dy
height = 550
width = int(height * dataAspect)
# Set download links for WMS, WCS or WFS and KML
links = wms_links(ogc_server_settings.public_url + 'wms?',
instance.typename.encode('utf-8'), instance.bbox_string,
instance.srid, height, width)
for ext, name, mime, wms_url in links:
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=wms_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
link_type='image',
)
)
if instance.storeType == "dataStore":
links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8'))
for ext, name, mime, wfs_url in links:
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=wfs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
url=wfs_url,
link_type='data',
)
)
elif instance.storeType == 'coverageStore':
#FIXME(Ariel): This works for public layers, does it work for restricted too?
# would those end up with no geotiff links, like, forever?
permissions = {}
permissions['anonymous'] = instance.get_gen_level(ANONYMOUS_USERS)
permissions['authenticated'] = instance.get_gen_level(AUTHENTICATED_USERS)
instance.set_gen_level(ANONYMOUS_USERS,'layer_readonly')
links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8'),
bbox=instance.bbox[:-1], crs=instance.bbox[-1], height=height, width=width)
for ext, name, mime, wcs_url in links:
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=wcs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
link_type='data',
)
)
instance.set_gen_level(ANONYMOUS_USERS,permissions['anonymous'])
instance.set_gen_level(AUTHENTICATED_USERS,permissions['authenticated'])
kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode({
'layers': instance.typename.encode('utf-8'),
'mode': "download"
})
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=kml_reflector_link_download,
defaults=dict(
extension='kml',
name=_("KML"),
mime='text/xml',
link_type='data',
)
)
kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode({
'layers': instance.typename.encode('utf-8'),
'mode': "refresh"
})
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=kml_reflector_link_view,
defaults=dict(
extension='kml',
name=_("View in Google Earth"),
mime='text/xml',
link_type='data',
)
)
tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url +
'layers=%s' % instance.typename.encode('utf-8') +
'&zoom={z}&x={x}&y={y}' +
'&format=image/png8'
)
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=tile_url,
defaults=dict(
extension='tiles',
name=_("Tiles"),
mime='image/png',
link_type='image',
)
)
html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url())
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=html_link_url,
defaults=dict(
extension='html',
name=instance.typename,
mime='text/html',
link_type='html',
)
)
#remove links that belong to and old address
for link in instance.link_set.all():
if not urlparse(ogc_server_settings.public_url).hostname == urlparse(link.url).hostname:
link.delete()
#Save layer attributes
set_attributes(instance)
#Save layer styles
set_styles(instance, gs_catalog)
def add_to_geoserver(self):
# Inspired (copied :) ) from https://groups.google.com/forum/#!msg/geonode-users/R-u57r8aECw/AuEpydZayfIJ # TODO : check license
# We connect to the catalog
gsUrl = settings.OGC_SERVER['default']['LOCATION'] + "rest"
gsUser = settings.OGC_SERVER['default']['USER']
gsPassword = settings.OGC_SERVER['default']['PASSWORD']
cat = Catalog(gsUrl, gsUser, gsPassword)
if cat is None:
raise Exception('unable to instantiate geoserver catalog')
# We get the workspace
ws = cat.get_workspace(settings.DEFAULT_WORKSPACE)
if ws is None:
raise Exception('workspace %s not found in geoserver' %
settings.DEFAULT_WORKSPACE)
# We get or create the datastore
store = cat.get_store(self.datastore_name, ws)
if store is None:
store = cat.create_datastore(self.datastore_name, ws)
store.connection_parameters.update(host="postgres",
port="5432",
database="postgres",
user="******",
passwd="postgres",
schema='offline_osm',
dbtype="postgis")
cat.save(store)
if store is None:
raise Exception('datastore %s not found in geoserver' %
self.datastore_name)
# We get or create each layer then register it into geonode
for layer in Command.layers:
layername = layer["name"]
self.stdout.write('Adding {} to geoserver...'.format(layername))
layer_exists = (not cat.get_layer(layername) is None)
if not layer_exists or not self.options['no_overwrite']:
self.stdout.write(
' layer does not exists or no_overwrite unset, we add...')
ft = cat.publish_featuretype(layername,
store,
'EPSG:4326',
srs='EPSG:4326')
if ft is None:
raise Exception('unable to publish layer %s' % layername)
ft.title = 'OpenStreetMap Offline - ' + layername.split(
'_')[-1]
ft.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.'
cat.save(ft)
self.stdout.write(
' adding the style for {}...'.format(layername))
# We get or create the workspace
style_path = os.path.join(os.path.dirname(__file__), '..',
'..', 'styles', layer["style_name"])
if not os.path.exists(style_path):
self.stdout.write(
' The file {} does not exist. No style will be applied for {}.'
.format(style_path, layername))
else:
cat.create_style(layer["style_name"],
open(style_path, 'r').read(),
overwrite=True,
workspace=settings.DEFAULT_WORKSPACE,
raw=True)
style = cat.get_style(layer["style_name"], ws)
if style is None:
raise Exception('style not found (%s)' %
(layer["style_name"]))
publishing = cat.get_layer(layername)
if publishing is None:
raise Exception('layer not found (%s)' % layerName)
publishing.default_style = style
cat.save(publishing)
self.stdout.write(
' registering {} into geonode...'.format(layername))
resource = cat.get_resource(layername, store, ws)
if resource is None:
raise Exception('resource not found (%s)' % layername)
layer, created = Layer.objects.get_or_create(name=layername)
layer.workspace = ws.name
layer.store = store.name
layer.storeType = store.resource_type
layer.typename = "%s:%s" % (ws.name.encode('utf-8'),
resource.name.encode('utf-8'))
layer.title = resource.title
layer.abstract = resource.abstract
layer.temporal_extent_start = self.import_timestamp
layer.temporal_extent_end = self.import_timestamp
try:
layer.save()
except TypeError as e:
# We ignore a specific error in Geonode resourcebase_post_save when settings.SITEURL has no host (relative URL)
self.stdout.write(
'TODO : report Geonode exception in resourcebase_post_save when settings.SITEURL has no host (relative URL) : '
+ str(e))
pass
if created:
layer.set_default_permissions()
else:
self.stdout.write(' layer already exists, we skip.')
# We get or create the laygroup
self.stdout.write('Adding layergroup to geoserver...')
layername = 'offline_osm'
layergroup_exists = (not cat.get_layergroup(layername) is None)
if not layer_exists or not self.options['no_overwrite']:
self.stdout.write(
' layer does not exists or no_overwrite unset, we add...')
layergroup = cat.get_layergroup(
layername, workspace=settings.DEFAULT_WORKSPACE)
if layergroup is None:
layergroup = cat.create_layergroup(
layername,
layers=layernames,
workspace=settings.DEFAULT_WORKSPACE)
if layergroup is None:
raise Exception('unable to publish layer %s' % layername)
layergroup.title = 'OpenStreetMap Offline'
layergroup.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.'
cat.save(layergroup)
else:
self.stdout.write(' layergroup already exists, we skip.')
# TODO : can we add layergroups to Geonode ?
self.stdout.write(
"laygroup won't be added to geonode (not supported yet)")
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
def testAbout(self):
about_html = self.cat.about()
self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer": continue
# looking for same name
ro = self.cat.get_resource(rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
# print "Updated layer: " + rd.name
count += 1
# print "Total updated layers: " + str(count)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assert_(self.cat.get_workspace("topp") is not None)
tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp")
self.cat.save(tas2)
self.assert_(self.cat.get_layergroup("tasmania_reloaded") is None)
self.assert_(self.cat.get_layergroup("tasmania_reloaded", "topp") is not None)
tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp")
self.assert_("tasmania_reloaded", tas2.name)
self.assert_(isinstance(tas2, LayerGroup))
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
self.assertEqual(tas2.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld", self.cat.get_style("population").filename)
self.assertEqual("population", self.cat.get_style("population").sld_name)
self.assert_(self.cat.get_style('non-existing-style') is None)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
try:
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.fail('expected exception')
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever', fre.message)
try:
self.cat.get_resource(workspace="best workspace ever",
store="best store ever",
name="best resource ever")
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever', fre.message)
class GeoserverLayers(object):
def __init__(self, gs_apirest, username, password):
try:
self.__catalog = Catalog(gs_apirest, username=username, password=password)
except Exception as err:
print("Geoserver API Rest Error: {0}".format(err))
def __createSldStyle(self, sldpath, stylename, ovwrt=False):
try:
styles_list = [st.name for st in self.__catalog.get_styles()]
if ovwrt or not stylename in styles_list:
with open(sldpath) as f:
self.__catalog.create_style(stylename, f.read(), overwrite=ovwrt)
else:
print("This style already exists...")
except Exception as err:
print("Geoserver API Rest Error creating new style: {0}".format(err))
def __rmvResources(self, rsrc):
"""
Removing resources
"""
self.__catalog.delete(rsrc)
self.__catalog.reload()
def rmvStyle(self, stylename):
"""
Remove style
"""
try:
style = self.__catalog.get_style(stylename)
self.__rmvResources(style)
except Exception as err:
print("Geoserver API Rest Error removing style: {0}".format(err))
def rmvDataStore(self, ds_store):
"""
Remove DataStore (with his layer)
"""
try:
lay_rm = self.__catalog.get_layer(ds_store)
self.__rmvResources(lay_rm)
str_rm = self.__catalog.get_store(ds_store)
self.__rmvResources(str_rm)
except Exception as err:
print("Geoserver API Rest Error removing data store: {0}".format(err))
def createGeoserverWMSLayer(self, data, ws_name, ds_name, stylename, sldpath, debug=False):
"""
Create Geoserver WMS layer
Status codes:
2 : "Geoserver layer successfully created"
-1 : "Workspace does not exist"
-2 : "Datastore already exists"
-3 : "Error creating Geoserver layer"
-4 : "File missing"
"""
try:
if not self.__catalog.get_workspace(ws_name):
print("Workspace does not exist")
return -1
ds_list = [i.name for i in self.__catalog.get_stores()]
if ds_name in ds_list:
print("Datastore already exists")
return -2
ds = self.__catalog.create_datastore(ds_name, ws_name)
ft = self.__catalog.create_featurestore(ds_name, workspace=ws_name,data=data)
print("Geoserver layer successfully created...")
self.__createSldStyle(sldpath, stylename)
lyr = self.__catalog.get_layer(ds_name)
lyr.enabled = True
lyr.default_style = stylename
self.__catalog.save(lyr)
if debug:
rsrc = self.__catalog.get_resource(ds_name, workspace=ws_name)
print("Data Store XML: {}".format(rsrc.href))
return 2
except Exception as err:
print("Geoserver API Rest Error creating new layer: {0}".format(err))
return -3
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assert_(self.cat.get_layer('import') is None)
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assert_(self.cat.get_layer('import') is not None)
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
self.cat.delete(ds)
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
def testCoverageStoreCreate(self):
ds = self.cat.create_coveragestore2("coverage_gsconfig")
ds.data_url = "file:data/mytiff.tiff"
self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
new_layer = self.cat.create_wmslayer(wmstest, wmsstore, layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
# TODO: Uploading WorldImage file no longer works???
# ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
# self.assertRaises(
# ConflictingDataError,
# lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# )
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assert_(f is not None)
self.assertEqual(f.name, fred.name)
def testWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assert_(jed is None)
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assert_(self.cat.get_style("jed", workspace="topp") is None)
# attempt creating new style
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assert_(f is not None)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp")
self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assert_(ned is not None)
self.assert_(zed is not None)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def _register_cascaded_layers(service, owner=None):
"""
Register layers for a cascading WMS
"""
if service.type == 'WMS' or service.type == "OWS":
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user, _password)
# Can we always assume that it is geonode?
# Should cascading layers have a separate workspace?
cascade_ws = cat.get_workspace(service.name)
if cascade_ws is None:
cascade_ws = cat.create_workspace(service.name, 'cascade')
try:
store = cat.get_store(service.name, cascade_ws)
except Exception:
store = cat.create_wmsstore(service.name, cascade_ws)
cat.save(store)
wms = WebMapService(service.base_url)
layers = list(wms.contents)
count = 0
for layer in layers:
lyr = cat.get_resource(layer, store, cascade_ws)
if lyr is None:
if service.type in ["WMS", "OWS"]:
resource = cat.create_wmslayer(cascade_ws, store, layer)
elif service.type == "WFS":
resource = cat.create_wfslayer(cascade_ws, store, layer)
if resource:
bbox = resource.latlon_bbox
cascaded_layer, created = Layer.objects.get_or_create(
typename="%s:%s" % (cascade_ws.name, resource.name),
service=service,
defaults={
"name": resource.name,
"workspace": cascade_ws.name,
"store": store.name,
"storeType": store.resource_type,
"title": resource.title or 'No title provided',
"abstract": resource.abstract
or 'No abstract provided',
"owner": None,
"uuid": str(uuid.uuid4()),
"bbox_x0": bbox[0],
"bbox_x1": bbox[1],
"bbox_y0": bbox[2],
"bbox_y1": bbox[3],
})
if created:
cascaded_layer.save()
if cascaded_layer is not None and cascaded_layer.bbox is None:
cascaded_layer._populate_from_gs(
gs_resource=resource)
cascaded_layer.set_default_permissions()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service, typename=cascaded_layer.name)
service_layer.layer = cascaded_layer
service_layer.title = cascaded_layer.title,
service_layer.description = cascaded_layer.abstract,
service_layer.styles = cascaded_layer.styles
service_layer.save()
count += 1
else:
logger.error(
"Resource %s from store %s could not be saved as layer"
% (layer, store.name))
message = "%d Layers Registered" % count
return_dict = {'status': 'ok', 'msg': message}
return HttpResponse(json.dumps(return_dict),
mimetype='application/json',
status=200)
elif service.type == 'WCS':
return HttpResponse('Not Implemented (Yet)', status=501)
else:
return HttpResponse('Invalid Service Type', status=400)
class wrap_geoserver:
""" Geoserver (gsconfig) wrapper """
def __init__(self,
geoserver_name,
username=username,
password=password,
easy=False):
if geoserver_name in list(REST.keys()):
self.path = REST[geoserver_name]
else:
self.path = geoserver_name
self.wms = self.path.replace("rest/", "wms")
self.name = geoserver_name
self.catalog = Catalog(self.path, username, password)
if not easy:
self.layers = []
self.layer_names = []
for layer in self.catalog.get_layers():
self.layers.append(layer)
self.layer_names.append(layer.name)
self.stores = [store for store in self.catalog.get_stores()]
self.store_names = [store.name for store in self.stores]
styles = []
self.workspaces = []
self.workspace_names = []
for workspace in self.catalog.get_workspaces():
styles = styles + self.catalog.get_styles(workspace)
self.workspace_names.append(workspace._name)
self.workspaces.append(workspace)
self.styles = styles + [
style for style in self.catalog.get_styles()
]
self.style_names = [style.name for style in self.styles]
def unpack(self, workspace_name, store_type="datastore"):
layers_and_styles = {}
features = []
workspace = self.get_workspace(workspace_name)
if store_type == "datastore":
store_url = workspace.datastore_url
elif store_type == "coveragestore":
store_url = workspace.coveragestore_url
else:
print("No correct store given")
for datastore in tqdm(get(store_url, "name")):
url = "{}workspaces/{}/datastores/{}".format(
self.path, workspace.name, datastore)
features = features + get(url, between_quotes=True)
for feature in features:
layer_name = os.path.basename(feature).split(".")[0]
self.get_layer(self.get_slug(workspace.name, layer_name))
layers_and_styles[layer_name] = self.layer.default_style
setattr(self, workspace_name + "_data", layers_and_styles)
return layers_and_styles
def get_layer(self, layer, easy=False):
self.layer = self.catalog.get_layer(layer)
if not easy:
self.resource = self.layer.resource
self.layer_name = self.layer.resource.name
self.sld_name = self.layer.default_style.name
self.sld_body = self.layer.default_style.sld_body
self.layer_latlon_bbox = self.layer.resource.latlon_bbox
self.layer_title = self.layer.resource.title
self.layer_abstract = self.layer.resource.abstract
def get_store(self, layer):
self.store = self.layer.resource._store
def get_resource(self):
self.resource = self.catalog.get_resource(self.layer.name, self.store)
def get_workspace(self, workspace_name):
self.workspace = self.catalog.get_workspace(workspace_name)
self.workspace_name = self.workspace._name
return self.workspace
def write_abstract(self, data, load_resource=True):
if load_resource:
self.get_resource()
self.resource.abstract = data
self.catalog.save(self.resource)
def write_title(self, title):
self.resource.title = title
self.catalog.save(self.resource)
def get_connection_parameters(self):
self.get_resource()
return self.resource.store.connection_parameters
def create_workspace(self, workspace_name):
workspace_exists = workspace_name in self.workspace_names
if not workspace_exists:
self.workspace = self.catalog.create_workspace(workspace_name)
else:
print("workspace already exists, using existing workspace")
self.workspace = self.catalog.get_workspace(workspace_name)
self.workspace_name = workspace_name
def create_postgis_datastore(self, store_name, workspace_name, pg_data):
try:
self.store = self.catalog.get_store(store_name,
self.workspace_name)
print("store within workspace exists, using existing store")
except Exception as e:
print(e)
ds = self.catalog.create_datastore(store_name, workspace_name)
ds.connection_parameters.update(
host=pg_data["host"],
port=pg_data["port"],
database=pg_data["database"],
user=pg_data["username"],
passwd=pg_data["password"],
dbtype="postgis",
schema="public",
)
self.save(ds)
self.store = self.catalog.get_store(store_name,
self.workspace_name)
self.store_name = store_name
def publish_layer(self,
layer_name,
workspace_name,
overwrite=False,
epsg="3857",
reload=False):
layer_exists = layer_name in self.layer_names
# if layer_name in self.workspace_layers[workspace_name]:
slug = self.get_slug(workspace_name, layer_name)
if overwrite and layer_exists:
print("Layer exists, deleting layer")
try:
self.layer = self.catalog.get_layer(slug)
self.delete(self.layer)
self.reload()
layer_exists = False
except Exception as e:
print(e)
print("Layer does not exist in workspace")
layer_exists = False
if not layer_exists:
feature_type = self.catalog.publish_featuretype(
layer_name,
self.store,
"EPSG:{}".format(str(epsg)),
srs="EPSG:{}".format(str(epsg)),
)
self.save(feature_type)
self.feature_type = feature_type
else:
print("layer already exists, using existing layer")
if reload:
self.get_layer(slug)
self.layer_name = layer_name
def publish_layergroup(self,
name,
layers,
styles=(),
bounds=None,
workspace=None):
layer_group = self.catalog.create_layergroup(name, layers, styles,
bounds, workspace)
self.save(layer_group)
def save(self, save_object):
return self.catalog.save(save_object)
def close(self):
self.catalog = None
def delete(self, delete_object):
self.catalog.delete(delete_object)
def reload(self):
self.catalog.reload()
def upload_shapefile(self, layer_name, shapefile_path):
path = shapefile_path.split(".shp")[0]
shapefile = shapefile_and_friends(path)
ft = self.catalog.create_featurestore(layer_name, shapefile,
self.workspace)
self.save(ft)
def upload_sld(self, sld_name, workspace_name, sld, overwrite=True):
style_exists = sld_name in self.style_names
if overwrite and style_exists:
print("Overwriting style")
style = self.catalog.get_style(sld_name, workspace_name)
self.delete(style)
self.reload()
style_exists = False
if not style_exists:
try:
self.catalog.create_style(sld_name, sld, False, workspace_name,
"sld11")
except Exception as e:
print(e)
style = self.catalog.get_style(sld_name, workspace_name)
self.delete(style)
self.reload()
self.catalog.create_style(sld_name, sld, False, workspace_name,
"sld10")
self.style_name = sld_name
else:
if style_exists:
print("Style already exists, using current style")
self.style_name = sld_name
def set_sld_for_layer(self,
workspace_name=None,
style_name=None,
use_custom=False):
if not use_custom:
workspace_name = self.workspace_name
style_name = self.style_name
self.style_slug = self.get_slug(workspace_name, style_name)
else:
if workspace_name is None:
self.style_slug = style_name
else:
self.style_slug = self.get_slug(workspace_name, style_name)
self.style = self.catalog.get_style(self.style_slug)
print("Setting {} for {}".format(self.style.name, self.layer.name))
self.layer.default_style = self.style
self.save(self.layer)
def get_slug(self, workspace, name):
return "{}:{}".format(workspace, name)
def get_slug_data(self, slug):
workspace_name = slug.split(":")[0]
layer_name = slug.split(":")[1]
return workspace_name, layer_name
def get_sld(self, layer_slug=None):
if layer_slug is None:
self.style = self.catalog.get_style(self.layer_slug)
else:
self.style = self.catalog.get_style(layer_slug)
self.sld_body = self.style.sld_body
return self.sld_body
def get_layer_workspace(self, layer_name):
return self.catalog.get_layer(layer_name).resource.workspace.name
def geoserver_post_save(instance, sender, **kwargs):
"""Save keywords to GeoServer
The way keywords are implemented require the layer
to be saved to the database before accessing them.
"""
url = "%srest" % settings.GEOSERVER_BASE_URL
try:
gs_catalog = Catalog(url, _user, _password)
gs_resource = gs_catalog.get_resource(instance.name)
except (FailedRequestError, EnvironmentError) as e:
msg = ('Could not connect to geoserver at "%s"'
'to save information for layer "%s"' % (
settings.GEOSERVER_BASE_URL, instance.name)
)
logger.warn(msg, e)
# If geoserver is not online, there is no need to continue
return
# If there is no resource returned it could mean one of two things:
# a) There is a syncronization problem in geoserver
# b) The unit tests are running and another geoserver is running in the
# background.
# For both cases it is sensible to stop processing the layer
if gs_resource is None:
logger.warn('Could not get geoserver resource for %s' % instance)
return
gs_resource.keywords = instance.keyword_list()
gs_catalog.save(gs_resource)
bbox = gs_resource.latlon_bbox
dx = float(bbox[1]) - float(bbox[0])
dy = float(bbox[3]) - float(bbox[2])
dataAspect = 1 if dy == 0 else dx / dy
height = 550
width = int(height * dataAspect)
# Set download links for WMS, WCS or WFS and KML
links = wms_links(settings.GEOSERVER_BASE_URL + 'wms?',
instance.typename, instance.bbox_string,
instance.srid, height, width)
for ext, name, mime, wms_url in links:
instance.link_set.get_or_create(url=wms_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
link_type='image',
)
)
if instance.storeType == "dataStore":
links = wfs_links(settings.GEOSERVER_BASE_URL + 'wfs?', instance.typename)
for ext, name, mime, wfs_url in links:
instance.link_set.get_or_create(url=wfs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
url=wfs_url,
link_type='data',
)
)
elif instance.storeType == 'coverageStore':
#FIXME(Ariel): This works for public layers, does it work for restricted too?
# would those end up with no geotiff links, like, forever?
links = wcs_links(settings.GEOSERVER_BASE_URL + 'wcs?', instance.typename)
for ext, name, mime, wcs_url in links:
instance.link_set.get_or_create(url=wcs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
link_type='data',
)
)
kml_reflector_link_download = settings.GEOSERVER_BASE_URL + "wms/kml?" + urllib.urlencode({
'layers': instance.typename,
'mode': "download"
})
instance.link_set.get_or_create(url=kml_reflector_link_download,
defaults=dict(
extension='kml',
name=_("KML"),
mime='text/xml',
link_type='data',
)
)
kml_reflector_link_view = settings.GEOSERVER_BASE_URL + "wms/kml?" + urllib.urlencode({
'layers': instance.typename,
'mode': "refresh"
})
instance.link_set.get_or_create(url=kml_reflector_link_view,
defaults=dict(
extension='kml',
name=_("View in Google Earth"),
mime='text/xml',
link_type='data',
)
)
tile_url = ('%sgwc/service/gmaps?' % settings.GEOSERVER_BASE_URL +
'layers=%s' % instance.typename +
'&zoom={z}&x={x}&y={y}' +
'&format=image/png8'
)
instance.link_set.get_or_create(url=tile_url,
defaults=dict(
extension='tiles',
name=_("Tiles"),
mime='text/png',
link_type='image',
)
)
#Save layer attributes
set_attributes(instance)
#Save layer styles
set_styles(instance, gs_catalog)
def createDataStore(name, filename, format="shapefile"):
cat = Catalog("{0}/rest/".format(geoserver_connection), geoserver_username,
geoserver_password)
ws = cat.get_workspace(workspace)
msg = ""
if format == "shapefile":
shapefile = shapefile_and_friends(filename)
try:
cat.create_featurestore(name, shapefile, workspace)
except ConflictingDataError as inst:
msg = str(inst)
except:
raise
resource = cat.get_resource(name, workspace=ws)
resource.projection = 'EPSG:4326'
cat.save(resource)
resource.projection_policy = 'REPROJECT_TO_DECLARED'
cat.save(resource)
resource.refresh()
bbox = resource.latlon_bbox[:4]
solr_geom = 'ENVELOPE({0},{1},{2},{3})'.format(bbox[0], bbox[1],
bbox[3], bbox[2])
return {
"solr_geom": solr_geom,
"msg": msg,
"resource_type": resource.resource_type
}
elif format == "image":
if cat.get_store(name):
newcs = cat.get_store(name, workspace=ws)
msg = "Geoserver datastore already existed. Update existing datastore."
else:
newcs = cat.create_coveragestore2(name, ws)
newcs.type = "GeoTIFF"
newcs.url = filename
cat.save(newcs)
# add coverage
url = "{0}/rest/workspaces/{1}/coveragestores/{2}/coverages.json"
url = url.format(geoserver_connection, ws.name, name)
headers = {"Content-Type": "application/json"}
coverageName = os.path.splitext(os.path.basename(filename))[0]
postdata = {
"coverage": {
"nativeCoverageName": coverageName,
"name": coverageName,
'projectionPolicy': 'REPROJECT_TO_DECLARED',
'srs': 'EPSG:4326'
}
}
requests.post(url,
json.dumps(postdata),
headers=headers,
auth=(geoserver_username, geoserver_password))
# Reproject
resource = cat.get_resource(name, workspace=ws)
url = "{0}/rest/workspaces/{1}/coveragestores/{2}/coverages/{2}?{3}"
parameters = "recalculate=nativebbox,latlonbbox"
url = url.format(geoserver_connection, ws.name, name, parameters)
headers = {"Content-Type": "application/x-www-form-urlencoded"}
requests.post(url,
headers=headers,
auth=(geoserver_username, geoserver_password))
# resource.refresh()
bbox = resource.latlon_bbox[:4]
solr_geom = 'ENVELOPE({0},{1},{2},{3})'.format(bbox[0], bbox[1],
bbox[3], bbox[2])
return {
"solr_geom": solr_geom,
"msg": msg,
"resource_type": resource.resource_type
}
return True
class UploaderTests(TestCase):
"""Basic checks to make sure pages load, etc.
"""
def create_datastore(self, connection, catalog):
"""Convenience method for creating a datastore.
"""
settings = connection.settings_dict
ds_name = settings['NAME']
params = {
'database': ds_name,
'passwd': settings['PASSWORD'],
'namespace': 'http://www.geonode.org/',
'type': 'PostGIS',
'dbtype': 'postgis',
'host': settings['HOST'],
'user': settings['USER'],
'port': settings['PORT'],
'enabled': 'True'
}
store = catalog.create_datastore(ds_name, workspace=self.workspace)
store.connection_parameters.update(params)
try:
catalog.save(store)
except FailedRequestError:
# assuming this is because it already exists
pass
return catalog.get_store(ds_name)
def create_user(self, username, password, **kwargs):
"""Convenience method for creating users.
"""
user, created = User.objects.get_or_create(username=username, **kwargs)
if created:
user.set_password(password)
user.save()
return user
def setUp(self):
self.assertTrue(
os.path.exists(_TEST_FILES_DIR),
'Test could not run due to missing test data at {0!r}'
.format(_TEST_FILES_DIR)
)
# These tests require geonode to be running on :80!
self.postgis = db.connections['datastore']
self.postgis_settings = self.postgis.settings_dict
self.admin_user = self.create_user('admin', 'admin', is_superuser=True)
self.non_admin_user = self.create_user('non_admin', 'non_admin')
self.catalog = Catalog(
ogc_server_settings.internal_rest,
*ogc_server_settings.credentials
)
if self.catalog.get_workspace('geonode') is None:
self.catalog.create_workspace('geonode', 'http://www.geonode.org/')
self.workspace = 'geonode'
self.datastore = self.create_datastore(self.postgis, self.catalog)
def tearDown(self):
"""Clean up geoserver.
"""
self.catalog.delete(self.datastore, recurse=True)
def import_file(self, path, configs=None):
"""Imports the file.
"""
if configs is None:
configs = []
self.assertTrue(os.path.exists(path))
# run ogr2ogr
ogr = OGRImport(path)
layers = ogr.handle(configuration_options=configs)
return layers
def generic_import(self, filename, configs=None):
if configs is None:
configs = [{'index': 0}]
path = test_file(filename)
results = self.import_file(path, configs=configs)
layer_results = []
for result in results:
if result[1].get('raster'):
layer_path = result[0]
layer_name = os.path.splitext(os.path.basename(layer_path))[0]
layer = Layer.objects.get(name=layer_name)
self.assertTrue(layer_path.endswith('.tif'))
self.assertTrue(os.path.exists(layer_path))
gdal_layer = gdal.OpenEx(layer_path)
self.assertTrue(gdal_layer.GetDriver().ShortName, 'GTiff')
layer_results.append(layer)
else:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
if not path.endswith('zip'):
self.assertGreaterEqual(
layer.attributes.count(),
DataSource(path)[0].num_fields
)
layer_results.append(layer)
return layer_results[0]
def generic_api_upload(self, filenames, configs=None):
"""Tests the import api.
"""
client = AdminClient()
client.login_as_non_admin()
# Don't accidentally iterate over given 'foo' as ['f', 'o', 'o'].
self.assertNotIsInstance(filenames, str)
# Upload Files
outfiles = []
for filename in filenames:
path = test_file(filename)
with open(path) as stream:
data = stream.read()
upload = SimpleUploadedFile(filename, data)
outfiles.append(upload)
response = client.post(
reverse('uploads-new-json'),
{'file': outfiles,
'json': json.dumps(configs)},
follow=True)
content = json.loads(response.content)
self.assertEqual(response.status_code, 200)
self.assertEqual(content['id'], 1)
# Configure Uploaded Files
upload_id = content['id']
upload_layers = UploadLayer.objects.filter(upload_id=upload_id)
for upload_layer in upload_layers:
for config in configs:
if config['upload_file_name'] == upload_layer.name:
payload = config['config']
url = '/importer-api/data-layers/{0}/configure/'.format(upload_layer.id)
response = client.post(
url, data=json.dumps(payload),
content_type='application/json'
)
self.assertTrue(response.status_code, 200)
url = '/importer-api/data-layers/{0}/'.format(upload_layer.id)
response = client.get(url, content_type='application/json')
self.assertTrue(response.status_code, 200)
return content
def generic_raster_import(self, filename, configs=None):
if configs is None:
configs = [{'index': 0}]
path = test_file(filename)
results = self.import_file(path, configs=configs)
layer_path = results[0][0]
layer_name = os.path.splitext(os.path.basename(layer_path))[0]
layer = Layer.objects.get(name=layer_name)
self.assertTrue(layer_path.endswith('.tif'))
self.assertTrue(os.path.exists(layer_path))
gdal_layer = gdal.OpenEx(layer_path)
self.assertTrue(gdal_layer.GetDriver().ShortName, 'GTiff')
return layer
def test_multi_upload(self):
"""Tests Uploading Multiple Files
"""
upload = self.generic_api_upload(
filenames=[
'boxes_with_year_field.zip',
'boxes_with_date.zip',
'point_with_date.geojson'
],
configs=[
{
'upload_file_name': 'boxes_with_year_field.shp',
'config': [{'index': 0}]
},
{
'upload_file_name': 'boxes_with_date.shp',
'config': [{'index': 0}]
},
{
'upload_file_name': 'point_with_date.geojson',
'config': [{'index': 0}]
}
]
)
self.assertEqual(9, upload['count'])
def test_upload_with_slds(self):
"""Tests Uploading sld
"""
upload = self.generic_api_upload(
filenames=[
'boxes_with_date.zip',
'boxes.sld',
'boxes1.sld'
],
configs=[
{
'upload_file_name': 'boxes_with_date.shp',
'config': [
{
'index': 0,
'default_style': 'boxes.sld',
'styles': ['boxes.sld', 'boxes1.sld']
}
]
}
]
)
self.assertEqual(6, upload['count'])
upload_id = upload['id']
upload_obj = UploadedData.objects.get(pk=upload_id)
uplayers = UploadLayer.objects.filter(upload=upload_id)
layer_id = uplayers[0].pk
upfiles_count = UploadFile.objects.filter(upload=upload_id).count()
self.assertEqual(6, upfiles_count)
# Warning: this assumes that Layer pks equal UploadLayer pks
layer = Layer.objects.get(pk=layer_id)
gslayer = self.catalog.get_layer(layer.name)
default_style = gslayer.default_style
# TODO: can we use public API or omit this?
self.catalog._cache.clear()
self.assertEqual('boxes.sld', default_style.filename)
def test_upload_with_metadata(self):
"""Tests Uploading metadata
"""
upload = self.generic_api_upload(
filenames=[
'boxes_with_date.zip',
'samplemetadata.xml',
],
configs=[
{
'upload_file_name': 'boxes_with_date.shp',
'config': [
{
'index': 0,
'metadata': 'samplemetadata.xml'
}
]
}
]
)
self.assertEqual(5, upload['count'])
upload_id = upload['id']
upload_obj = UploadedData.objects.get(pk=upload_id)
# TODO: why did we get upload_obj?
uplayers = UploadLayer.objects.filter(upload=upload_id)
layer_id = uplayers[0].pk
upfiles_count = UploadFile.objects.filter(upload=upload_id).count()
self.assertEqual(5, upfiles_count)
layer = Layer.objects.get(pk=layer_id)
self.assertEqual(layer.language, 'eng')
self.assertEqual(layer.title, 'Old_Americas_LSIB_Polygons_Detailed_2013Mar')
def test_raster(self):
"""Exercise raster import.
"""
layer = self.generic_raster_import(
'test_grid.tif',
configs=[
{
'index': 0
}
]
)
self.assertTrue(layer.name.startswith('test_grid'))
def test_box_with_year_field(self):
"""Tests the import of test_box_with_year_field.
"""
layer = self.generic_import(
'boxes_with_year_field.shp',
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date(self):
"""Tests the import of test_boxes_with_date.
"""
layer = self.generic_import(
'boxes_with_date.shp',
configs=[
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_gpkg(self):
"""Tests the import of test_boxes_with_date.gpkg.
"""
layer = self.generic_import(
'boxes_with_date.gpkg',
configs=[
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_plus_raster_gpkg_by_index(self):
"""Tests the import of multilayer vector + raster geopackage using index
"""
layer = self.generic_import(
'boxes_plus_raster.gpkg',
configs=[
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
},
{'index': 1},
{'index': 2},
{'index': 3},
{'index': 4},
{'index': 5},
{'index': 6},
{'index': 7},
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_csv(self):
"""Tests a CSV with WKT polygon.
"""
layer = self.generic_import(
'boxes_with_date.csv',
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_csv_missing_features(self):
"""Test csv that has some rows without geoms and uses ISO-8859 (Latin 4) encoding.
"""
self.generic_import(
'missing-features.csv',
configs=[
{'index': 0}
]
)
def test_boxes_with_iso_date(self):
"""Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import(
'boxes_with_date_iso_date.shp',
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_duplicate_imports(self):
"""Import the same layer twice to ensure file names increment properly.
"""
path = test_file('boxes_with_date_iso_date.zip')
ogr = OGRImport(path)
layers1 = ogr.handle({'index': 0, 'name': 'test'})
layers2 = ogr.handle({'index': 0, 'name': 'test'})
self.assertEqual(layers1[0][0], 'test')
self.assertEqual(layers2[0][0], 'test0')
def test_launder(self):
"""Ensure the launder function works as expected.
"""
self.assertEqual(launder('tm_world_borders_simpl_0.3'), 'tm_world_borders_simpl_0_3')
self.assertEqual(launder('Testing#'), 'testing_')
self.assertEqual(launder(' '), '_')
def test_boxes_with_date_iso_date_zip(self):
"""Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import(
'boxes_with_date_iso_date.zip',
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_dates_bc(self):
"""Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import(
'boxes_with_dates_bc.shp',
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_point_with_date(self):
"""Tests the import of point_with_date.geojson
"""
layer = self.generic_import(
'point_with_date.geojson',
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
# OGR will name geojson layers 'ogrgeojson' we rename to the path basename
self.assertTrue(layer.name.startswith('point_with_date'))
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_end_date(self):
"""Tests the import of test_boxes_with_end_date.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import(
'boxes_with_end_date.shp',
configs=[
{
'index': 0,
'convert_to_date': ['date', 'enddate'],
'start_date': 'date',
'end_date': 'enddate',
'configureTime': True
}
]
)
date_attr = get_layer_attr(layer, 'date_as_date')
end_date_attr = get_layer_attr(layer, 'enddate_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
self.assertEqual(end_date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute
)
self.generic_time_check(
layer,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute
)
def test_us_states_kml(self):
"""Tests the import of us_states_kml.
This layer has a date and an end date field that are typed correctly.
"""
# TODO: Support time in kmls.
layer = self.generic_import(
'us_states.kml',
configs=[
{
'index': 0
}
]
)
self.assertEqual(layer.name.lower(), "us_states")
def test_mojstrovka_gpx(self):
"""Tests the import of mojstrovka.gpx.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import(
'mojstrovka.gpx',
configs=[
{
'index': 0,
'convert_to_date': ['time'],
'configureTime': True
}
]
)
date_attr = get_layer_attr(layer, 'time_as_date')
self.assertEqual(date_attr.attribute_type, u'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def generic_time_check(self, layer, attribute=None, end_attribute=None):
"""Convenience method to run generic tests on time layers.
"""
# TODO: can we use public API or omit this?
self.catalog._cache.clear()
resource = self.catalog.get_resource(
layer.name, store=layer.store, workspace=self.workspace
)
time_info = resource.metadata['time']
self.assertEqual('LIST', time_info.presentation)
self.assertEqual(True, time_info.enabled)
self.assertEqual(attribute, time_info.attribute)
self.assertEqual(end_attribute, time_info.end_attribute)
def test_us_shootings_csv(self):
"""Tests the import of US_Shootings.csv.
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
path = test_file('US_Shootings.csv')
layer = self.generic_import(
path,
configs=[
{
'index': 0,
'convert_to_date': ['Date']
}
]
)
self.assertTrue(layer.name.startswith('us_shootings'))
date_field = 'date'
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_field
)
self.generic_time_check(layer, attribute=date_field)
def test_sitins(self):
"""Tests the import of US_Civil_Rights_Sitins0.csv
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
layer = self.generic_import(
'US_Civil_Rights_Sitins0.csv',
configs=[
{
'index': 0, 'convert_to_date': ['Date']
}
]
)
self.assertEqual(layer.name.lower(), 'us_civil_rights_sitins0')
def get_layer_names(self, path):
"""Gets layer names from a data source.
"""
data_source = DataSource(path)
return [layer.name for layer in data_source]
def test_gdal_import(self):
path = test_file('point_with_date.geojson')
self.generic_import(
path,
configs=[
{
'index': 0,
'convert_to_date': ['date']
}
]
)
def test_wfs(self):
"""Tests the import from a WFS Endpoint
"""
wfs = 'WFS:http://demo.boundlessgeo.com/geoserver/wfs'
ogr = OGRImport(wfs)
configs = [
{'layer_name': 'og:bugsites'},
{'layer_name': 'topp:states'}
]
layers = ogr.handle(configuration_options=configs)
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def test_arcgisjson(self):
"""Tests the import from a WFS Endpoint
"""
endpoint = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services/Hydrography/Watershed173811/FeatureServer/0/query?where=objectid+%3D+objectid&outfields=*&f=json'
ogr = OGRImport(endpoint)
configs = [{'index':0}]
layers = ogr.handle(configuration_options=configs)
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def test_file_add_view(self):
"""Tests the file_add_view.
"""
client = AdminClient()
# test login required for this view
request = client.get(reverse('uploads-new'))
self.assertEqual(request.status_code, 302)
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context['request'].path, reverse('uploads-list'))
self.assertEqual(len(response.context['object_list']), 1)
upload = response.context['object_list'][0]
self.assertEqual(upload.user.username, 'non_admin')
self.assertEqual(upload.file_type, 'GeoJSON')
self.assertTrue(upload.uploadlayer_set.all())
self.assertEqual(upload.state, 'UPLOADED')
self.assertIsNotNone(upload.name)
uploaded_file = upload.uploadfile_set.first()
self.assertTrue(os.path.exists(uploaded_file.file.path))
with open(test_file('empty_file.geojson')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
self.assertEqual(response.status_code, 200)
self.assertIn('file', response.context_data['form'].errors)
def test_file_add_view_as_json(self):
"""Tests the file_add_view.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(
reverse('uploads-new-json'),
{'file': stream},
follow=True
)
self.assertEqual(response.status_code, 200)
self.assertIn('application/json', response.get('Content-Type', ''))
content = json.loads(response.content)
self.assertIn('state', content)
self.assertIn('id', content)
def test_describe_fields(self):
"""Tests the describe fields functionality.
"""
path = test_file('US_Shootings.csv')
with GDALInspector(path) as inspector:
layers = inspector.describe_fields()
self.assertTrue(layers[0]['layer_name'], 'us_shootings')
self.assertEqual([n['name'] for n in layers[0]['fields']], ['Date', 'Shooter', 'Killed',
'Wounded', 'Location', 'City',
'Longitude', 'Latitude'])
self.assertEqual(layers[0]['feature_count'], 203)
def test_gdal_file_type(self):
"""Tests the describe fields functionality.
"""
filenames = {
'US_Shootings.csv': 'CSV',
'point_with_date.geojson': 'GeoJSON',
'mojstrovka.gpx': 'GPX',
'us_states.kml': 'KML',
'boxes_with_year_field.shp': 'ESRI Shapefile',
'boxes_with_date_iso_date.zip': 'ESRI Shapefile'
}
from .models import NoDataSourceFound
try:
for filename, file_type in sorted(filenames.items()):
path = test_file(filename)
with GDALInspector(path) as inspector:
self.assertEqual(inspector.file_type(), file_type)
except NoDataSourceFound:
logging.exception('No data source found in: {0}'.format(path))
raise
def test_configure_view(self):
"""Tests the configuration view.
"""
path = test_file('point_with_date.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
client = AdminClient()
client.login_as_non_admin()
with open(path) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
upload = response.context['object_list'][0]
payload = [
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {
'users': {
'test': new_user_perms,
'AnonymousUser': [
'change_layer_data',
'download_resourcebase',
'view_resourcebase'
]
}
}
}
]
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id),
data=json.dumps(payload),
content_type='application/json'
)
self.assertTrue(response.status_code, 200)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.srid, 'EPSG:4326')
self.assertEqual(first_layer.store, self.datastore.name)
self.assertEqual(first_layer.storeType, 'dataStore')
self.assertTrue(first_layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertEqual(
Layer.objects.all()[0].owner.username,
self.non_admin_user.username
)
perms = first_layer.get_all_level_info()
user = User.objects.get(username=self.non_admin_user.username)
# check user permissions
expected_perms = [
u'publish_resourcebase',
u'change_resourcebase_permissions',
u'delete_resourcebase',
u'change_resourcebase',
u'change_resourcebase_metadata',
u'download_resourcebase',
u'view_resourcebase',
u'change_layer_style',
u'change_layer_data'
]
for perm in expected_perms:
self.assertIn(perm, perms['users'][user])
self.assertTrue(perms['users'][new_user])
self.assertIn(
'change_resourcebase_permissions',
perms['users'][new_user]
)
self.assertIn(
'change_layer_data',
perms['users'][User.objects.get(username='******')]
)
catalog_layer = self.catalog.get_layer(first_layer.name)
self.assertIn('time', catalog_layer.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, first_layer)
def test_configure_view_convert_date(self):
"""Tests the configure view with a dataset that needs to be converted to a date.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('US_Shootings.csv')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
upload = response.context['object_list'][0]
payload = [
{
'index': 0,
'convert_to_date': ['Date'],
'start_date': 'Date',
'configureTime': True,
'editable': True
}
]
response = client.get(
'/importer-api/data-layers/{0}/configure/'.format(upload.id)
)
self.assertEqual(response.status_code, 405)
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id)
)
self.assertEqual(response.status_code, 400)
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id),
data=json.dumps(payload),
content_type='application/json'
)
self.assertTrue(response.status_code, 200)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.srid, 'EPSG:4326')
self.assertEqual(first_layer.store, self.datastore.name)
self.assertEqual(first_layer.storeType, 'dataStore')
self.assertTrue(first_layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertTrue(first_layer.attributes.filter(attribute='date'), 'xsd:dateTime')
catalog_layer = self.catalog.get_layer(first_layer.name)
self.assertIn('time', catalog_layer.resource.metadata)
# ensure a user who does not own the upload cannot configure an import from it.
client.logout()
client.login_as_admin()
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id)
)
self.assertEqual(response.status_code, 404)
def test_list_api(self):
client = AdminClient()
response = client.get('/importer-api/data/')
self.assertEqual(response.status_code, 401)
client.login_as_non_admin()
response = client.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
admin = User.objects.get(username=self.admin_user.username)
non_admin = User.objects.get(username=self.non_admin_user.username)
path = test_file('US_Shootings.csv')
with open(path, 'rb') as stream:
uploaded_file = SimpleUploadedFile('test_data', stream.read())
admin_upload = UploadedData.objects.create(state='Admin test', user=admin)
admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file))
non_admin_upload = UploadedData.objects.create(state='Non admin test', user=non_admin)
non_admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file))
client.login_as_admin()
response = client.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 2)
response = client.get('/importer-api/data/?user__username=admin')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 1)
def test_layer_list_api(self):
client = AdminClient()
response = client.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 401)
client.login_as_non_admin()
response = client.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 200)
def test_delete_from_non_admin_api(self):
"""Ensure users can delete their data.
"""
client = AdminClient()
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
self.assertEqual(UploadedData.objects.all().count(), 1)
upload_id = UploadedData.objects.first().id
response = client.delete('/importer-api/data/{0}/'.format(upload_id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def test_delete_from_admin_api(self):
"""Ensure that administrators can delete data that isn't theirs.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
self.assertEqual(UploadedData.objects.all().count(), 1)
client.logout()
client.login_as_admin()
upload_id = UploadedData.objects.first().id
response = client.delete('/importer-api/data/{0}/'.format(upload_id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def naming_an_import(self):
"""Tests providing a name in the configuration options.
"""
client = AdminClient()
client.login_as_non_admin()
name = 'point-with-a-date'
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'name': name,
'editable': True
}
response = client.post(
'/importer-api/data-layers/1/configure/',
data=json.dumps(payload),
content_type='application/json'
)
self.assertEqual(response.status_code, 200)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.title, name.replace('-', '_'))
def test_api_import(self):
"""Tests the import api.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(
reverse('uploads-new'),
{'file': stream},
follow=True
)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True
}
self.assertIsInstance(
UploadLayer.objects.first().configuration_options,
dict
)
response = client.get('/importer-api/data-layers/1/')
self.assertEqual(response.status_code, 200)
response = client.post(
'/importer-api/data-layers/1/configure/',
data=json.dumps([payload]),
content_type='application/json'
)
self.assertEqual(response.status_code, 200)
self.assertIn('task', response.content)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.srid, 'EPSG:4326')
self.assertEqual(first_layer.store, self.datastore.name)
self.assertEqual(first_layer.storeType, 'dataStore')
self.assertTrue(first_layer.attributes[1].attribute_type, 'xsd:dateTime')
catalog_layer = self.catalog.get_layer(first_layer.name)
self.assertIn('time', catalog_layer.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, first_layer)
self.assertTrue(UploadLayer.objects.first().task_id)
def test_valid_file_extensions(self):
"""Test the file extension validator.
"""
for extension in load_handler(OSGEO_IMPORTER, 'test.txt').valid_extensions:
filename = 'test.{0}'.format(extension)
upload = SimpleUploadedFile(filename, '')
self.assertIsNone(validate_file_extension(upload))
with self.assertRaises(ValidationError):
validate_file_extension(SimpleUploadedFile('test.txt', ''))
def test_no_geom(self):
"""Test the file extension validator.
"""
with self.assertRaises(ValidationError):
validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)'))
# This should pass (geom type is unknown)
validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)'))
def test_numeric_overflow(self):
"""Regression test for numeric field overflows in shapefiles.
# https://trac.osgeo.org/gdal/ticket/5241
"""
self.generic_import(
'Walmart.zip',
configs=[
{
'configureTime':False,
'convert_to_date': ['W1_OPENDAT'],
'editable': True,
'index':0,
'name': 'Walmart',
'start_date': 'W1_OPENDAT'
}
]
)
def test_multipolygon_shapefile(self):
"""Tests shapefile with multipart polygons.
"""
self.generic_import('PhoenixFirstDues.zip', configs=[{'index': 0}])
def test_istanbul(self):
"""Tests shapefile with multipart polygons and non-WGS84 SR.
"""
result = self.generic_import(
'Istanbul.zip',
configs=[
{'index': 0}
]
)
feature_type = self.catalog.get_resource(result.name)
self.assertEqual(feature_type.projection, 'EPSG:32635')
def test_gwc_handler(self):
"""Tests the GeoWebCache handler
"""
layer = self.generic_import(
'boxes_with_date.shp',
configs=[
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}
]
)
gwc = GeoWebCacheHandler(None)
gs_layer = self.catalog.get_layer(layer.name)
self.assertTrue(gwc.time_enabled(gs_layer))
gs_layer.fetch()
payload = self.catalog.http.request(gwc.gwc_url(gs_layer))
self.assertIn('regexParameterFilter', payload[1])
self.assertEqual(int(payload[0]['status']), 200)
# Don't configure time, ensure everything still works
layer = self.generic_import(
'boxes_with_date_iso_date.shp',
configs=[
{
'index': 0
}
]
)
gs_layer = self.catalog.get_layer(layer.name)
self.catalog._cache.clear()
gs_layer.fetch()
payload = self.catalog.http.request(gwc.gwc_url(gs_layer))
self.assertNotIn('regexParameterFilter', payload[1])
self.assertEqual(int(payload[0]['status']), 200)
def test_utf8(self):
"""Tests utf8 characters in attributes
"""
path = test_file('china_provinces.shp')
layer = self.generic_import(path)
ogr = OGRImport(path)
datastore, _ = ogr.open_target_datastore(ogr.target_store)
sql = (
"select NAME_CH from {0} where NAME_PY = 'An Zhou'"
.format(layer.name)
)
result = datastore.ExecuteSQL(sql)
feature = result.GetFeature(0)
self.assertEqual(feature.GetField('name_ch'), '安州')
def test_non_converted_date(self):
"""Test converting a field as date.
"""
results = self.generic_import(
'TM_WORLD_BORDERS_2005.zip',
configs=[
{
'index': 0,
'start_date': 'Year',
'configureTime': True
}
]
)
layer = self.catalog.get_layer(results.typename)
self.assertIn('time', layer.resource.metadata)
self.assertEqual('year', layer.resource.metadata['time'].attribute)
def test_fid_field(self):
"""
Regression test for preserving an FID field when target layer supports
it but source does not.
"""
self.generic_import(
'noaa_paleoclimate.zip',
configs=[
{
'index': 0
}
]
)
def test_csv_with_wkb_geometry(self):
"""Exercise import of CSV files with multiple geometries.
"""
filenames = [
'police_csv.csv',
'police_csv_nOGC.csv',
'police_csv_noLatLon.csv',
'police_csv_WKR.csv',
'police_csv_nFID.csv',
'police_csv_nOGFID.csv',
'police_csv_noWKB.csv'
]
for filename in filenames:
self.generic_import(
filename,
{
'configureTime': True,
'convert_to_date': ['date_time'],
'editable': True,
'index': 0,
'name': filename.lower(),
'permissions': {
'users':{
'AnonymousUser': [
'change_layer_data',
'download_resourcebase',
'view_resourcebase'
]
}
},
'start_date':'date_time',
}
)
class UploaderTests(TestCase):
"""Basic checks to make sure pages load, etc.
"""
def create_datastore(self, connection, catalog):
"""Convenience method for creating a datastore.
"""
settings = connection.settings_dict
ds_name = settings['NAME']
params = {
'database': ds_name,
'passwd': settings['PASSWORD'],
'namespace': 'http://www.geonode.org/',
'type': 'PostGIS',
'dbtype': 'postgis',
'host': settings['HOST'],
'user': settings['USER'],
'port': settings['PORT'],
'enabled': 'True'
}
store = catalog.create_datastore(ds_name, workspace=self.workspace)
store.connection_parameters.update(params)
try:
catalog.save(store)
except FailedRequestError:
# assuming this is because it already exists
pass
return catalog.get_store(ds_name)
def create_user(self, username, password, **kwargs):
"""Convenience method for creating users.
"""
user, created = User.objects.get_or_create(username=username, **kwargs)
if created:
user.set_password(password)
user.save()
return user
def setUp(self):
self.assertTrue(
os.path.exists(_TEST_FILES_DIR),
'Test could not run due to missing test data at {0!r}'.format(
_TEST_FILES_DIR))
# These tests require geonode to be running on :80!
self.postgis = db.connections['datastore']
self.postgis_settings = self.postgis.settings_dict
self.admin_user = self.create_user('admin', 'admin', is_superuser=True)
self.non_admin_user = self.create_user('non_admin', 'non_admin')
self.catalog = Catalog(ogc_server_settings.internal_rest,
*ogc_server_settings.credentials)
if self.catalog.get_workspace('geonode') is None:
self.catalog.create_workspace('geonode', 'http://www.geonode.org/')
self.workspace = 'geonode'
self.datastore = self.create_datastore(self.postgis, self.catalog)
def tearDown(self):
"""Clean up geoserver.
"""
self.catalog.delete(self.datastore, recurse=True)
def import_file(self, path, configs=None):
"""Imports the file.
"""
if configs is None:
configs = []
self.assertTrue(os.path.exists(path))
# run ogr2ogr
ogr = OGRImport(path)
layers = ogr.handle(configuration_options=configs)
return layers
def generic_import(self, filename, configs=None):
if configs is None:
configs = [{'index': 0}]
path = test_file(filename)
results = self.import_file(path, configs=configs)
layer_results = []
for result in results:
if result[1].get('raster'):
layer_path = result[0]
layer_name = os.path.splitext(os.path.basename(layer_path))[0]
layer = Layer.objects.get(name=layer_name)
self.assertTrue(layer_path.endswith('.tif'))
self.assertTrue(os.path.exists(layer_path))
gdal_layer = gdal.OpenEx(layer_path)
self.assertTrue(gdal_layer.GetDriver().ShortName, 'GTiff')
layer_results.append(layer)
else:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
if not path.endswith('zip'):
self.assertGreaterEqual(layer.attributes.count(),
DataSource(path)[0].num_fields)
layer_results.append(layer)
return layer_results[0]
def generic_api_upload(self, filenames, configs=None):
"""Tests the import api.
"""
client = AdminClient()
client.login_as_non_admin()
# Don't accidentally iterate over given 'foo' as ['f', 'o', 'o'].
self.assertNotIsInstance(filenames, str)
# Upload Files
outfiles = []
for filename in filenames:
path = test_file(filename)
with open(path) as stream:
data = stream.read()
upload = SimpleUploadedFile(filename, data)
outfiles.append(upload)
response = client.post(reverse('uploads-new-json'), {
'file': outfiles,
'json': json.dumps(configs)
},
follow=True)
content = json.loads(response.content)
self.assertEqual(response.status_code, 200)
self.assertEqual(content['id'], 1)
# Configure Uploaded Files
upload_id = content['id']
upload_layers = UploadLayer.objects.filter(upload_id=upload_id)
for upload_layer in upload_layers:
for config in configs:
if config['upload_file_name'] == os.path.basename(
upload_layer.name):
payload = config['config']
url = '/importer-api/data-layers/{0}/configure/'.format(
upload_layer.id)
response = client.post(url,
data=json.dumps(payload),
content_type='application/json')
self.assertEqual(response.status_code, 200)
url = '/importer-api/data-layers/{0}/'.format(
upload_layer.id)
response = client.get(url, content_type='application/json')
self.assertEqual(response.status_code, 200)
return content
def generic_raster_import(self, filename, configs=None):
if configs is None:
configs = [{'index': 0}]
path = test_file(filename)
results = self.import_file(path, configs=configs)
layer_path = results[0][0]
layer_name = os.path.splitext(os.path.basename(layer_path))[0]
layer = Layer.objects.get(name=layer_name)
self.assertTrue(layer_path.endswith('.tif'))
self.assertTrue(os.path.exists(layer_path))
gdal_layer = gdal.OpenEx(layer_path)
self.assertTrue(gdal_layer.GetDriver().ShortName, 'GTiff')
return layer
def test_multi_upload(self):
"""Tests Uploading Multiple Files
"""
# Number of layers in each file
upload_layer_counts = [1, 1, 1]
upload = self.generic_api_upload(filenames=[
'boxes_with_year_field.zip', 'boxes_with_date.zip',
'point_with_date.geojson'
],
configs=[{
'upload_file_name':
'boxes_with_year_field.shp',
'config': [{
'index': 0
}]
}, {
'upload_file_name':
'boxes_with_date.shp',
'config': [{
'index': 0
}]
}, {
'upload_file_name':
'point_with_date.geojson',
'config': [{
'index': 0
}]
}])
self.assertEqual(Layer.objects.count(), sum(upload_layer_counts))
self.assertEqual(9, upload['count'])
def test_upload_with_slds(self):
"""Tests Uploading sld
"""
upload = self.generic_api_upload(
filenames=['boxes_with_date.zip', 'boxes.sld', 'boxes1.sld'],
configs=[{
'upload_file_name':
'boxes_with_date.shp',
'config': [{
'index': 0,
'default_style': 'boxes.sld',
'styles': ['boxes.sld', 'boxes1.sld']
}]
}])
self.assertEqual(6, upload['count'])
upload_id = upload['id']
uplayers = UploadLayer.objects.filter(upload=upload_id)
layer_id = uplayers[0].pk
upfiles_count = UploadFile.objects.filter(upload=upload_id).count()
self.assertEqual(6, upfiles_count)
# Warning: this assumes that Layer pks equal UploadLayer pks
layer = Layer.objects.get(pk=layer_id)
gslayer = self.catalog.get_layer(layer.name)
default_style = gslayer.default_style
# TODO: can we use public API or omit this?
self.catalog._cache.clear()
self.assertEqual('boxes.sld', default_style.filename)
def test_upload_with_metadata(self):
"""Tests Uploading metadata
"""
upload = self.generic_api_upload(filenames=[
'boxes_with_date.zip',
'samplemetadata.xml',
],
configs=[{
'upload_file_name':
'boxes_with_date.shp',
'config': [{
'index':
0,
'metadata':
'samplemetadata.xml'
}]
}])
self.assertEqual(5, upload['count'])
upload_id = upload['id']
uplayers = UploadLayer.objects.filter(upload=upload_id)
layer_id = uplayers[0].pk
upfiles_count = UploadFile.objects.filter(upload=upload_id).count()
self.assertEqual(5, upfiles_count)
layer = Layer.objects.get(pk=layer_id)
self.assertEqual(layer.language, 'eng')
self.assertEqual(layer.title,
'Old_Americas_LSIB_Polygons_Detailed_2013Mar')
def test_geotiff_raster(self):
"""Exercise GeoTIFF raster import.
"""
layer = self.generic_raster_import('test_grid.tif',
configs=[{
'index': 0
}])
self.assertTrue(layer.name.startswith('test_grid'))
def test_nitf_raster(self):
"""Tests NITF raster import
"""
layer = self.generic_raster_import('test_nitf.nitf')
self.assertTrue(layer.name.startswith('test_nitf'))
def test_box_with_year_field(self):
"""Tests the import of test_box_with_year_field.
"""
layer = self.generic_import('boxes_with_year_field.shp',
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date(self):
"""Tests the import of test_boxes_with_date.
"""
layer = self.generic_import('boxes_with_date.shp',
configs=[{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_gpkg(self):
"""Tests the import of test_boxes_with_date.gpkg.
"""
layer = self.generic_import('boxes_with_date.gpkg',
configs=[{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_plus_raster_gpkg_by_index(self):
""" Tests the import of multilayer vector + tile geopackage using index, treating tile layers
as rasters.
Tile layers are now treated by default as a distinct layer type.
This test forces them to still be treated as rasters and should be
removed once tests for vector/tile geopackage files are in place.
"""
layer = self.generic_import('boxes_plus_raster.gpkg',
configs=[
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
},
{
'index': 1
},
{
'index': 2
},
{
'index': 3
},
{
'index': 4
},
{
'index': 5
},
{
'index': 6,
'layer_type': 'raster'
},
{
'index': 7,
'layer_type': 'raster'
},
])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_csv(self):
"""Tests a CSV with WKT polygon.
"""
layer = self.generic_import('boxes_with_date.csv',
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_csv_missing_features(self):
"""Test csv that has some rows without geoms and uses ISO-8859 (Latin 4) encoding.
"""
self.generic_import('missing-features.csv', configs=[{'index': 0}])
def test_boxes_with_iso_date(self):
"""Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.shp',
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_duplicate_imports(self):
"""Import the same layer twice to ensure file names increment properly.
"""
path = test_file('boxes_with_date_iso_date.zip')
ogr = OGRImport(path)
layers1 = ogr.handle({'index': 0, 'name': 'test'})
layers2 = ogr.handle({'index': 0, 'name': 'test'})
self.assertEqual(layers1[0][0], 'test')
self.assertEqual(layers2[0][0], 'test0')
def test_launder(self):
"""Ensure the launder function works as expected.
"""
self.assertEqual(launder('tm_world_borders_simpl_0.3'),
'tm_world_borders_simpl_0_3')
self.assertEqual(launder('Testing#'), 'testing_')
self.assertEqual(launder(' '), '_')
def test_boxes_with_date_iso_date_zip(self):
"""Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.zip',
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_dates_bc(self):
"""Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('boxes_with_dates_bc.shp',
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_point_with_date(self):
"""Tests the import of point_with_date.geojson
"""
layer = self.generic_import('point_with_date.geojson',
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
# OGR will name geojson layers 'ogrgeojson' we rename to the path basename
self.assertTrue(layer.name.startswith('point_with_date'))
date_attr = get_layer_attr(layer, 'date_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_end_date(self):
"""Tests the import of test_boxes_with_end_date.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('boxes_with_end_date.shp',
configs=[{
'index':
0,
'convert_to_date': ['date', 'enddate'],
'start_date':
'date',
'end_date':
'enddate',
'configureTime':
True
}])
date_attr = get_layer_attr(layer, 'date_as_date')
end_date_attr = get_layer_attr(layer, 'enddate_as_date')
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
self.assertEqual(end_date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
self.generic_time_check(layer,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
def test_us_states_kml(self):
"""Tests the import of us_states_kml.
This layer has a date and an end date field that are typed correctly.
"""
# TODO: Support time in kmls.
layer = self.generic_import('us_states.kml', configs=[{'index': 0}])
self.assertEqual(layer.name.lower(), "us_states")
def test_mojstrovka_gpx(self):
"""Tests the import of mojstrovka.gpx.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('mojstrovka.gpx',
configs=[{
'index': 0,
'convert_to_date': ['time'],
'configureTime': True
}])
date_attr = get_layer_attr(layer, 'time_as_date')
self.assertEqual(date_attr.attribute_type, u'xsd:dateTime')
configure_time(self.catalog.get_layer(layer.name).resource,
attribute=date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute)
def generic_time_check(self, layer, attribute=None, end_attribute=None):
"""Convenience method to run generic tests on time layers.
"""
# TODO: can we use public API or omit this?
self.catalog._cache.clear()
resource = self.catalog.get_resource(layer.name,
store=layer.store,
workspace=self.workspace)
time_info = resource.metadata['time']
self.assertEqual('LIST', time_info.presentation)
self.assertEqual(True, time_info.enabled)
self.assertEqual(attribute, time_info.attribute)
self.assertEqual(end_attribute, time_info.end_attribute)
def test_us_shootings_csv(self):
"""Tests the import of US_Shootings.csv.
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
path = test_file('US_Shootings.csv')
layer = self.generic_import(path,
configs=[{
'index': 0,
'convert_to_date': ['Date']
}])
self.assertTrue(layer.name.startswith('us_shootings'))
date_field = 'date'
configure_time(self.catalog.get_layer(layer.name).resource,
attribute=date_field)
self.generic_time_check(layer, attribute=date_field)
def test_sitins(self):
"""Tests the import of US_Civil_Rights_Sitins0.csv
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
layer = self.generic_import('US_Civil_Rights_Sitins0.csv',
configs=[{
'index': 0,
'convert_to_date': ['Date']
}])
self.assertEqual(layer.name.lower(), 'us_civil_rights_sitins0')
def get_layer_names(self, path):
"""Gets layer names from a data source.
"""
data_source = DataSource(path)
return [layer.name for layer in data_source]
def test_gdal_import(self):
path = test_file('point_with_date.geojson')
self.generic_import(path,
configs=[{
'index': 0,
'convert_to_date': ['date']
}])
def test_wfs(self):
"""Tests the import from a WFS Endpoint
"""
wfs = 'WFS:http://demo.geo-solutions.it/geoserver/tiger/wfs'
ogr = OGRImport(wfs)
configs = [
{
'layer_name': 'tiger:giant_polygon'
},
{
'layer_name': 'tiger:poi'
},
]
layers = ogr.handle(configuration_options=configs)
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def test_arcgisjson(self):
"""Tests the import from a WFS Endpoint
"""
endpoint = 'http://sampleserver6.arcgisonline.com/arcgis/rest/services/Water_Network/FeatureServer/16/query'\
'?where=objectid=326&outfields=*&f=json'
ogr = OGRImport(endpoint)
configs = [{'index': 0}]
layers = ogr.handle(configuration_options=configs)
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def test_file_add_view(self):
"""Tests the file_add_view.
"""
client = AdminClient()
# test login required for this view
request = client.get(reverse('uploads-new'))
self.assertEqual(request.status_code, 302)
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context['request'].path,
reverse('uploads-list'))
self.assertEqual(len(response.context['object_list']), 1)
upload = response.context['object_list'][0]
self.assertEqual(upload.user.username, 'non_admin')
self.assertEqual(upload.file_type, 'GeoJSON')
self.assertTrue(upload.uploadlayer_set.all())
self.assertEqual(upload.state, 'UPLOADED')
self.assertIsNotNone(upload.name)
uploaded_file = upload.uploadfile_set.first()
self.assertTrue(os.path.exists(uploaded_file.file.path))
with open(test_file('empty_file.geojson')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('file', response.context_data['form'].errors)
def test_file_add_view_as_json(self):
"""Tests the file_add_view.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(reverse('uploads-new-json'),
{'file': stream},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('application/json', response.get('Content-Type', ''))
content = json.loads(response.content)
self.assertIn('state', content)
self.assertIn('id', content)
def test_describe_fields(self):
"""Tests the describe fields functionality.
"""
path = test_file('US_Shootings.csv')
with GDALInspector(path) as inspector:
layers = inspector.describe_fields()
self.assertTrue(layers[0]['layer_name'], 'us_shootings')
self.assertEqual([n['name'] for n in layers[0]['fields']], [
'Date', 'Shooter', 'Killed', 'Wounded', 'Location', 'City',
'Longitude', 'Latitude'
])
self.assertEqual(layers[0]['feature_count'], 203)
def test_gdal_file_type(self):
"""Tests the describe fields functionality.
"""
filenames = {
'US_Shootings.csv': {'CSV'},
'point_with_date.geojson': {'GeoJSON'},
'mojstrovka.gpx': {'GPX'},
'us_states.kml': {'LIBKML', 'KML'},
'boxes_with_year_field.shp': {'ESRI Shapefile'},
'boxes_with_date_iso_date.zip': {'ESRI Shapefile'}
}
from osgeo_importer.models import NoDataSourceFound
try:
for filename, file_type in sorted(filenames.items()):
path = test_file(filename)
with GDALInspector(path) as inspector:
self.assertIn(inspector.file_type(), file_type)
except NoDataSourceFound:
logging.exception('No data source found in: {0}'.format(path))
raise
def test_configure_view(self):
"""Tests the configuration view.
"""
path = test_file('point_with_date.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
client = AdminClient()
client.login_as_non_admin()
with open(path) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
upload = response.context['object_list'][0]
payload = [{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {
'users': {
'test':
new_user_perms,
'AnonymousUser': [
'change_layer_data', 'download_resourcebase',
'view_resourcebase'
]
}
}
}]
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id),
data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.srid, 'EPSG:4326')
self.assertEqual(first_layer.store, self.datastore.name)
self.assertEqual(first_layer.storeType, 'dataStore')
self.assertTrue(first_layer.attributes[1].attribute_type,
'xsd:dateTime')
self.assertEqual(Layer.objects.all()[0].owner.username,
self.non_admin_user.username)
perms = first_layer.get_all_level_info()
user = User.objects.get(username=self.non_admin_user.username)
# check user permissions
expected_perms = [
u'publish_resourcebase', u'change_resourcebase_permissions',
u'delete_resourcebase', u'change_resourcebase',
u'change_resourcebase_metadata', u'download_resourcebase',
u'view_resourcebase', u'change_layer_style', u'change_layer_data'
]
for perm in expected_perms:
self.assertIn(perm, perms['users'][user])
self.assertTrue(perms['users'][new_user])
self.assertIn('change_resourcebase_permissions',
perms['users'][new_user])
self.assertIn(
'change_layer_data',
perms['users'][User.objects.get(username='******')])
catalog_layer = self.catalog.get_layer(first_layer.name)
self.assertIn('time', catalog_layer.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, first_layer)
def test_configure_view_convert_date(self):
"""Tests the configure view with a dataset that needs to be converted to a date.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('US_Shootings.csv')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
upload = response.context['object_list'][0]
payload = [{
'index': 0,
'convert_to_date': ['Date'],
'start_date': 'Date',
'configureTime': True,
'editable': True
}]
response = client.get(
'/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 405)
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 400)
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id),
data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.srid, 'EPSG:4326')
self.assertEqual(first_layer.store, self.datastore.name)
self.assertEqual(first_layer.storeType, 'dataStore')
self.assertTrue(first_layer.attributes[1].attribute_type,
'xsd:dateTime')
self.assertTrue(first_layer.attributes.filter(attribute='date'),
'xsd:dateTime')
catalog_layer = self.catalog.get_layer(first_layer.name)
self.assertIn('time', catalog_layer.resource.metadata)
# ensure a user who does not own the upload cannot configure an import from it.
client.logout()
client.login_as_admin()
response = client.post(
'/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 404)
def test_list_api(self):
client = AdminClient()
response = client.get('/importer-api/data/')
self.assertEqual(response.status_code, 401)
client.login_as_non_admin()
response = client.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
admin = User.objects.get(username=self.admin_user.username)
non_admin = User.objects.get(username=self.non_admin_user.username)
path = test_file('US_Shootings.csv')
with open(path, 'rb') as stream:
uploaded_file = SimpleUploadedFile('test_data', stream.read())
admin_upload = UploadedData.objects.create(state='Admin test',
user=admin)
admin_upload.uploadfile_set.add(
UploadFile.objects.create(file=uploaded_file))
non_admin_upload = UploadedData.objects.create(
state='Non admin test', user=non_admin)
non_admin_upload.uploadfile_set.add(
UploadFile.objects.create(file=uploaded_file))
client.login_as_admin()
response = client.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 2)
response = client.get('/importer-api/data/?user__username=admin')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 1)
def test_layer_list_api(self):
client = AdminClient()
response = client.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 401)
client.login_as_non_admin()
response = client.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 200)
def test_delete_from_non_admin_api(self):
"""Ensure users can delete their data.
"""
client = AdminClient()
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
upload_id = UploadedData.objects.first().id
response = client.delete('/importer-api/data/{0}/'.format(upload_id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def test_delete_from_admin_api(self):
"""Ensure that administrators can delete data that isn't theirs.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
client.logout()
client.login_as_admin()
upload_id = UploadedData.objects.first().id
response = client.delete('/importer-api/data/{0}/'.format(upload_id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def naming_an_import(self):
"""Tests providing a name in the configuration options.
"""
client = AdminClient()
client.login_as_non_admin()
name = 'point-with-a-date'
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'name': name,
'editable': True
}
response = client.post('/importer-api/data-layers/1/configure/',
data=json.dumps(payload),
content_type='application/json')
self.assertEqual(response.status_code, 200)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.title, name.replace('-', '_'))
def test_api_import(self):
"""Tests the import api.
"""
client = AdminClient()
client.login_as_non_admin()
with open(test_file('point_with_date.geojson')) as stream:
response = client.post(reverse('uploads-new'), {'file': stream},
follow=True)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True
}
self.assertIsInstance(
UploadLayer.objects.first().configuration_options, dict)
response = client.get('/importer-api/data-layers/1/')
self.assertEqual(response.status_code, 200)
response = client.post('/importer-api/data-layers/1/configure/',
data=json.dumps([payload]),
content_type='application/json')
self.assertEqual(response.status_code, 200)
self.assertIn('task', response.content)
first_layer = Layer.objects.all()[0]
self.assertEqual(first_layer.srid, 'EPSG:4326')
self.assertEqual(first_layer.store, self.datastore.name)
self.assertEqual(first_layer.storeType, 'dataStore')
self.assertTrue(first_layer.attributes[1].attribute_type,
'xsd:dateTime')
catalog_layer = self.catalog.get_layer(first_layer.name)
self.assertIn('time', catalog_layer.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, first_layer)
self.assertTrue(UploadLayer.objects.first().task_id)
def test_valid_file_extensions(self):
"""Test the file extension validator.
"""
for extension in load_handler(OSGEO_IMPORTER,
'test.txt').valid_extensions:
filename = 'test.{0}'.format(extension)
upload = SimpleUploadedFile(filename, '')
self.assertIsNone(validate_file_extension(upload))
with self.assertRaises(ValidationError):
validate_file_extension(SimpleUploadedFile('test.txt', ''))
def test_no_geom(self):
"""Test the file extension validator.
"""
with self.assertRaises(ValidationError):
validate_inspector_can_read(
SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)'))
# This should pass (geom type is unknown)
validate_inspector_can_read(
SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)'))
def test_numeric_overflow(self):
"""Regression test for numeric field overflows in shapefiles.
# https://trac.osgeo.org/gdal/ticket/5241
"""
self.generic_import('Walmart.zip',
configs=[{
'configureTime': False,
'convert_to_date': ['W1_OPENDAT'],
'editable': True,
'index': 0,
'name': 'Walmart',
'start_date': 'W1_OPENDAT'
}])
def test_multipolygon_shapefile(self):
"""Tests shapefile with multipart polygons.
"""
self.generic_import('PhoenixFirstDues.zip', configs=[{'index': 0}])
def test_istanbul(self):
"""Tests shapefile with multipart polygons and non-WGS84 SR.
"""
result = self.generic_import('Istanbul.zip', configs=[{'index': 0}])
feature_type = self.catalog.get_resource(result.name)
self.assertEqual(feature_type.projection, 'EPSG:32635')
def test_houston_tx_annexations(self):
"""Tests Shapefile with originally unsupported EPSG Code.
"""
result = self.generic_import('HoustonTXAnnexations.zip',
configs=[{
'index': 0
}])
feature_type = self.catalog.get_resource(result.name)
self.assertEqual(feature_type.projection, 'EPSG:2278')
def test_gwc_handler(self):
"""Tests the GeoWebCache handler
"""
layer = self.generic_import('boxes_with_date.shp',
configs=[{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
gwc = GeoWebCacheHandler(None)
gs_layer = self.catalog.get_layer(layer.name)
self.assertTrue(gwc.time_enabled(gs_layer))
gs_layer.fetch()
payload = self.catalog.http.request(gwc.gwc_url(gs_layer))
self.assertIn('regexParameterFilter', payload[1])
self.assertEqual(int(payload[0]['status']), 200)
# Don't configure time, ensure everything still works
layer = self.generic_import('boxes_with_date_iso_date.shp',
configs=[{
'index': 0
}])
gs_layer = self.catalog.get_layer(layer.name)
self.catalog._cache.clear()
gs_layer.fetch()
payload = self.catalog.http.request(gwc.gwc_url(gs_layer))
self.assertNotIn('regexParameterFilter', payload[1])
self.assertEqual(int(payload[0]['status']), 200)
def test_utf8(self):
"""Tests utf8 characters in attributes
"""
path = test_file('china_provinces.shp')
layer = self.generic_import(path)
ogr = OGRImport(path)
datastore, _ = ogr.open_target_datastore(ogr.target_store)
sql = ("select NAME_CH from {0} where NAME_PY = 'An Zhou'".format(
layer.name))
result = datastore.ExecuteSQL(sql)
feature = result.GetFeature(0)
self.assertEqual(feature.GetField('name_ch'), '安州')
def test_non_converted_date(self):
"""Test converting a field as date.
"""
results = self.generic_import('TM_WORLD_BORDERS_2005.zip',
configs=[{
'index': 0,
'start_date': 'Year',
'configureTime': True
}])
layer = self.catalog.get_layer(results.typename)
self.assertIn('time', layer.resource.metadata)
self.assertEqual('year', layer.resource.metadata['time'].attribute)
def test_fid_field(self):
"""
Regression test for preserving an FID field when target layer supports
it but source does not.
"""
self.generic_import('noaa_paleoclimate.zip', configs=[{'index': 0}])
def test_csv_with_wkb_geometry(self):
"""Exercise import of CSV files with multiple geometries.
"""
filenames = [
'police_csv.csv', 'police_csv_nOGC.csv', 'police_csv_noLatLon.csv',
'police_csv_WKR.csv', 'police_csv_nFID.csv',
'police_csv_nOGFID.csv', 'police_csv_noWKB.csv'
]
for filename in filenames:
self.generic_import(
filename, {
'configureTime': True,
'convert_to_date': ['date_time'],
'editable': True,
'index': 0,
'name': filename.lower(),
'permissions': {
'users': {
'AnonymousUser': [
'change_layer_data', 'download_resourcebase',
'view_resourcebase'
]
}
},
'start_date': 'date_time',
})
class UploaderTests(MapStoryTestMixin):
"""
Basic checks to make sure pages load, etc.
"""
def create_datastore(self, connection, catalog):
settings = connection.settings_dict
params = {
'database': settings['NAME'],
'passwd': settings['PASSWORD'],
'namespace': 'http://www.geonode.org/',
'type': 'PostGIS',
'dbtype': 'postgis',
'host': settings['HOST'],
'user': settings['USER'],
'port': settings['PORT'],
'enabled': "True"
}
store = catalog.create_datastore(settings['NAME'],
workspace=self.workspace)
store.connection_parameters.update(params)
try:
catalog.save(store)
except FailedRequestError:
# assuming this is because it already exists
pass
return catalog.get_store(settings['NAME'])
def setUp(self):
if not os.path.exists(
os.path.join(os.path.split(__file__)[0], 'test_ogr')):
self.skipTest('Skipping test due to missing test data.')
# These tests require geonode to be running on :80!
self.postgis = db.connections['datastore']
self.postgis_settings = self.postgis.settings_dict
self.username, self.password = self.create_user('admin',
'admin',
is_superuser=True)
self.non_admin_username, self.non_admin_password = self.create_user(
'non_admin', 'non_admin')
self.cat = Catalog(ogc_server_settings.internal_rest,
*ogc_server_settings.credentials)
self.workspace = 'geonode'
self.datastore = self.create_datastore(self.postgis, self.cat)
def tearDown(self):
"""
Clean up geoserver.
"""
self.cat.delete(self.datastore, recurse=True)
def generic_import(self, file, configuration_options=[{'index': 0}]):
f = file
filename = os.path.join(os.path.dirname(__file__), 'test_ogr', f)
res = self.import_file(filename,
configuration_options=configuration_options)
layer = Layer.objects.get(name=res[0][0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
if not filename.endswith('zip'):
self.assertTrue(
layer.attributes.count() >= DataSource(filename)[0].num_fields)
# make sure we have at least one dateTime attribute
self.assertTrue('xsd:dateTime' or 'xsd:date'
in [n.attribute_type for n in layer.attributes.all()])
return layer
def test_box_with_year_field(self):
"""
Tests the import of test_box_with_year_field.
"""
layer = self.generic_import('boxes_with_year_field.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date(self):
"""
Tests the import of test_boxes_with_date.
"""
layer = self.generic_import('boxes_with_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date'],
'start_date':
'date',
'configureTime':
True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_csv(self):
"""
Tests a CSV with WKT polygon.
"""
layer = self.generic_import('boxes_with_date.csv',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_iso_date(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_iso_date_zip(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.zip',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_dates_bc(self):
"""
Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('boxes_with_dates_bc.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_point_with_date(self):
"""
Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('point_with_date.geojson',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
# OGR will name geojson layers 'ogrgeojson' we rename to the path basename
self.assertTrue(layer.name.startswith('point_with_date'))
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_end_date(self):
"""
Tests the import of test_boxes_with_end_date.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('boxes_with_end_date.shp')
date_attr = filter(lambda attr: attr.attribute == 'date',
layer.attributes)[0]
end_date_attr = filter(lambda attr: attr.attribute == 'enddate',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:date')
self.assertEqual(end_date_attr.attribute_type, 'xsd:date')
configure_time(self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
self.generic_time_check(layer,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
def test_us_states_kml(self):
"""
Tests the import of us_states_kml.
This layer has a date and an end date field that are typed correctly.
"""
# TODO: Support time in kmls.
layer = self.generic_import('us_states.kml')
def test_mojstrovka_gpx(self):
"""
Tests the import of mojstrovka.gpx.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('mojstrovka.gpx')
date_attr = filter(lambda attr: attr.attribute == 'time',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, u'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute)
def generic_time_check(self, layer, attribute=None, end_attribute=None):
"""
Convenience method to run generic tests on time layers.
"""
self.cat._cache.clear()
resource = self.cat.get_resource(layer.name,
store=layer.store,
workspace=self.workspace)
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual(attribute, timeInfo.attribute)
self.assertEqual(end_attribute, timeInfo.end_attribute)
def test_us_shootings_csv(self):
"""
Tests the import of US_shootings.csv.
"""
filename = 'US_shootings.csv'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename)
layer = self.generic_import(filename,
configuration_options=[{
'index':
0,
'convert_to_date': ['Date']
}])
self.assertEqual(layer.name, 'us_shootings')
date_field = 'date_as_date'
configure_time(self.cat.get_layer(layer.name).resource,
attribute=date_field)
self.generic_time_check(layer, attribute=date_field)
def test_sitins(self):
"""
Tests the import of US_shootings.csv.
"""
filename = 'US_Civil_Rights_Sitins0.csv'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename)
layer = self.generic_import(f,
configuration_options=[{
'index':
0,
'convert_to_date': ['Date']
}])
def get_layer_names(self, in_file):
"""
Gets layer names from a data source.
"""
ds = DataSource(in_file)
return map(lambda layer: layer.name, ds)
def test_gdal_import(self):
filename = 'point_with_date.geojson'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename)
self.generic_import(filename,
configuration_options=[{
'index': 0,
'convert_to_date': ['date']
}])
def import_file(self, in_file, configuration_options=[]):
"""
Imports the file.
"""
self.assertTrue(os.path.exists(in_file))
# run ogr2ogr
gi = GDALImport(in_file)
layers = gi.handle(configuration_options=configuration_options)
return layers
@staticmethod
def createFeatureType(catalog, datastore, name):
"""
Exposes a PostGIS feature type in geoserver.
"""
headers = {"Content-type": "application/xml"}
data = "<featureType><name>{name}</name></featureType>".format(
name=name)
url = datastore.href.replace(".xml",
'/featuretypes.xml'.format(name=name))
headers, response = catalog.http.request(url, "POST ", data, headers)
return response
def test_create_vrt(self):
"""
Tests the create_vrt function.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'US_shootings.csv')
vrt = create_vrt(f)
vrt.seek(0)
output = vrt.read()
self.assertTrue('name="US_shootings"' in output)
self.assertTrue(
'<SrcDataSource>{0}</SrcDataSource>'.format(f) in output)
self.assertTrue(
'<GeometryField encoding="PointFromColumns" x="Longitude" y="Latitude" />'
.format(f) in output)
self.assertEqual(os.path.splitext(vrt.name)[1], '.vrt')
def test_file_add_view(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson')
c = AdminClient()
# test login required for this view
request = c.get(reverse('uploads-new'))
self.assertEqual(request.status_code, 302)
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context['request'].path,
reverse('uploads-list'))
self.assertTrue(len(response.context['object_list']) == 1)
upload = response.context['object_list'][0]
self.assertEqual(upload.user.username, 'non_admin')
self.assertEqual(upload.file_type, 'GeoJSON')
self.assertTrue(upload.uploadlayer_set.all())
self.assertEqual(upload.state, 'UPLOADED')
self.assertIsNotNone(upload.name)
uploaded_file = upload.uploadfile_set.first()
self.assertTrue(os.path.exists(uploaded_file.file.path))
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'empty_file.geojson')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('file', response.context_data['form'].errors)
def test_describe_fields(self):
"""
Tests the describe fields functionality.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'us_shootings.csv')
fields = None
with GDALInspector(f) as f:
layers = f.describe_fields()
self.assertTrue(layers[0]['name'], 'us_shootings')
self.assertEqual([n['name'] for n in layers[0]['fields']], [
'Date', 'Shooter', 'Killed', 'Wounded', 'Location', 'City',
'Longitude', 'Latitude'
])
self.assertEqual(layers[0]['feature_count'], 203)
def test_gdal_file_type(self):
"""
Tests the describe fields functionality.
"""
files = (
(os.path.join(os.path.dirname(__file__), 'test_ogr',
'us_shootings.csv'), 'CSV'),
(os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson'), 'GeoJSON'),
(os.path.join(os.path.dirname(__file__), 'test_ogr',
'mojstrovka.gpx'), 'GPX'),
(os.path.join(os.path.dirname(__file__), 'test_ogr',
'us_states.kml'), 'KML'),
(os.path.join(os.path.dirname(__file__), 'test_ogr',
'boxes_with_year_field.shp'), 'ESRI Shapefile'),
(os.path.join(os.path.dirname(__file__), 'test_ogr',
'boxes_with_date_iso_date.zip'), 'ESRI Shapefile'),
)
for path, file_type in files:
with GDALInspector(path) as f:
self.assertEqual(f.file_type(), file_type)
def test_configure_view(self):
"""
Tests the configuration view.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
upload = response.context['object_list'][0]
payload = [{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True
}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id),
data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertEqual(Layer.objects.all()[0].owner.username,
self.non_admin_username)
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
def test_configure_view_convert_date(self):
"""
Tests the configure view with a dataset that needs to be converted to a date.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'US_shootings.csv')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
upload = response.context['object_list'][0]
payload = [{
'index': 0,
'convert_to_date': ['Date'],
'start_date': 'Date',
'configureTime': True,
'editable': True
}]
response = c.get('/importer-api/data-layers/{0}/configure/'.format(
upload.id))
self.assertEqual(response.status_code, 405)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id))
self.assertEqual(response.status_code, 400)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id),
data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertTrue(layer.attributes.filter(attribute='date_as_date'),
'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
# ensure a user who does not own the upload cannot configure an import from it.
c.logout()
c.login_as_admin()
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id))
self.assertEqual(response.status_code, 404)
def test_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
admin = User.objects.get(username=self.username)
non_admin = User.objects.get(username=self.non_admin_username)
from .models import UploadFile
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'US_shootings.csv')
with open(f, 'rb') as f:
uploaded_file = SimpleUploadedFile('test_data', f.read())
admin_upload = UploadedData.objects.create(state='Admin test',
user=admin)
admin_upload.uploadfile_set.add(
UploadFile.objects.create(file=uploaded_file))
non_admin_upload = UploadedData.objects.create(
state='Non admin test', user=non_admin)
non_admin_upload.uploadfile_set.add(
UploadFile.objects.create(file=uploaded_file))
c.login_as_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 2)
response = c.get('/importer-api/data/?user__username=admin')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 1)
def test_layer_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 200)
def test_delete_from_non_admin_api(self):
"""
Ensure users can delete their data.
"""
c = AdminClient()
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def test_delete_from_admin_api(self):
"""
Ensure that administrators can delete data that isn't theirs.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
c.logout()
c.login_as_admin()
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def naming_an_import(self):
"""
Tests providing a name in the configuration options.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
name = 'point-with-a-date'
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'name': name,
'editable': True
}
response = c.post('/importer-api/data-layers/1/configure/',
data=json.dumps(payload),
content_type='application/json')
layer = Layer.objects.all()[0]
self.assertEqual(layer.title, name.replace('-', '_'))
def test_api_import(self):
"""
Tests the import api.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr',
'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True
}
self.assertTrue(
isinstance(UploadLayer.objects.first().configuration_options,
dict))
response = c.get('/importer-api/data-layers/1/')
self.assertEqual(response.status_code, 200)
response = c.post('/importer-api/data-layers/1/configure/',
data=json.dumps([payload]),
content_type='application/json')
self.assertEqual(response.status_code, 200)
self.assertTrue('task' in response.content)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
self.assertTrue(UploadLayer.objects.first().task_id)
def test_valid_file_extensions(self):
"""
Test the file extension validator.
"""
for extension in IMPORTER_VALID_EXTENSIONS:
self.assertIsNone(
validate_file_extension(
SimpleUploadedFile('test.{0}'.format(extension), '')))
with self.assertRaises(ValidationError):
validate_file_extension(SimpleUploadedFile('test.txt', ''))
def test_no_geom(self):
"""
Test the file extension validator.
"""
with self.assertRaises(ValidationError):
validate_inspector_can_read(
SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)'))
# This should pass (geom type is unknown)
validate_inspector_can_read(
SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)'))
def test_numeric_overflow(self):
"""
Regression test for numeric field overflows in shapefiles.
# https://trac.osgeo.org/gdal/ticket/5241
"""
self.generic_import('Walmart.zip',
configuration_options=[{
'index': 0,
'convert_to_date': []
}])
def test_outside_bounds_regression(self):
"""
Regression where layers with features outside projection bounds fail.
"""
self.generic_import('Spring_2015.zip',
configuration_options=[{
'index': 0
}])
resource = self.cat.get_layer('spring_2015').resource
self.assertEqual(resource.latlon_bbox,
('-180.0', '180.0', '-90.0', '90.0', 'EPSG:4326'))
def test_gwc_handler(self):
"""
Tests the GeoWebCache handler
"""
layer = self.generic_import('boxes_with_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date'],
'start_date':
'date',
'configureTime':
True
}])
gwc = GeoWebCacheHandler(None)
gs_layer = self.cat.get_layer(layer.name)
self.assertTrue(gwc.time_enabled(gs_layer))
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertTrue('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
# Don't configure time, ensure everything still works
layer = self.generic_import('boxes_with_date_iso_date.shp',
configuration_options=[{
'index': 0
}])
gs_layer = self.cat.get_layer(layer.name)
self.cat._cache.clear()
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertFalse('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
logging.info("Uploading " + str(len(files)) + " styles")
for f in files:
with open(styleDir + f, 'r') as style:
styleName = f.rstrip('.xml')
cat.create_style(styleName, style.read(), workspace=workspace, overwrite=True)
styles.append(workspace + ':' + styleName)
break
# assign styles to created layers and enable timeDimension
layers = cat.get_layers()
for layer in layers:
# if layer.dom is not None:
if layer.name.startswith(workspace):
# GetStyleName
layerName = layer.resource.name
# Set Default Style (timeIndependend)
layer.default_style = workspace + ":" + layerName
layer.styles = [s for s in styles if s.startswith(workspace + ":" + layerName)]
cat.save(layer)
# get coverage to activate time Dimension
from geoserver.support import DimensionInfo
coverage = cat.get_resource(layerName, projectName, workspace=workspace)
timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None)
coverage.metadata = ({'time': timeInfo})
coverage.CheckAuxiliaryMetadata=True
cat.save(coverage)
if 'removeOutputFiles' in cfg['general'] and cfg['general']['removeOutputFiles'] is not False:
shutil.rmtree(cfg['general']['workdir'] + '/outputFiles/' + projectName + '/')
logging.info("Deleted outputFiles")
logging.info('App can be started at: ' + cfg['frontend']['path'] + '/projects/' + projectName + '/index.html')
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assert_(self.cat.get_layer('import') is None)
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assert_(self.cat.get_layer('import') is not None)
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
self.cat.delete(ds)
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(
ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
def testCoverageStoreCreate(self):
ds = self.cat.create_coveragestore2("coverage_gsconfig")
ds.data_url = "file:data/mytiff.tiff"
self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(
['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before,
str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after,
str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before,
str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after,
str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats,
str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after,
str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
# sanitize the layer name - validation will fail on newer geoservers
name = layer.replace(':', '_')
new_layer = self.cat.create_wmslayer(wmstest,
wmsstore,
name,
nativeName=layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
changed_layer = added_layers[0]
self.assertEqual(True, changed_layer.advertised)
self.assertEqual(True, changed_layer.enabled)
changed_layer.advertised = False
changed_layer.enabled = False
self.cat.save(changed_layer)
self.cat._cache.clear()
changed_layer = wmsstore.get_resources()[0]
changed_layer.fetch()
self.assertEqual(False, changed_layer.advertised)
self.assertEqual(False, changed_layer.enabled)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars,
sf)
self.assert_(
self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError, lambda: self.cat.create_featurestore(
"states_test", shapefile_plus_sidecars, sf))
self.assertRaises(
UploadError, lambda: self.cat.create_coveragestore(
"states_raster_test", shapefile_plus_sidecars, sf))
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf))
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
# TODO: Uploading WorldImage file no longer works???
# ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
# self.assertRaises(
# ConflictingDataError,
# lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# )
self.assertRaises(
UploadError, lambda: self.cat.create_featurestore(
"Pk50095_vector", tiffdata, sf))
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff))
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles
if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred",
open("test/ted.sld").read(),
overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assert_(f is not None)
self.assertEqual(f.name, fred.name)
def testWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("jed",
open("test/fred.sld").read(),
workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assert_(jed is None)
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
self.cat.create_style("jed",
open("test/ted.sld").read(),
overwrite=True,
workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assert_(self.cat.get_style("jed", workspace="topp") is None)
# attempt creating new style
self.cat.create_style("jed",
open("test/fred.sld").read(),
workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assert_(f is not None)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("ned",
open("test/fred.sld").read(),
overwrite=True,
workspace="topp")
self.cat.create_style("zed",
open("test/ted.sld").read(),
overwrite=True,
workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assert_(ned is not None)
self.assert_(zed is not None)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def testTimeDimension(self):
sf = self.cat.get_workspace("sf")
files = shapefile_and_friends(
os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date"))
self.cat.create_featurestore("boxes_with_end_date", files, sf)
get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer(
'boxes_with_end_date').resource
# configure time as LIST
resource = get_resource()
timeInfo = DimensionInfo("time",
"true",
"LIST",
None,
"ISO8601",
None,
attribute="date")
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual("date", timeInfo.attribute)
self.assertEqual("ISO8601", timeInfo.units)
# disable time dimension
timeInfo = resource.metadata['time']
timeInfo.enabled = False
# since this is an xml property, it won't get written unless we modify it
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(False, timeInfo.enabled)
# configure with interval, end_attribute and enable again
timeInfo.enabled = True
timeInfo.presentation = 'DISCRETE_INTERVAL'
timeInfo.resolution = '3 days'
timeInfo.end_attribute = 'enddate'
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(True, timeInfo.enabled)
self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation)
self.assertEqual('3 days', timeInfo.resolution_str())
self.assertEqual('enddate', timeInfo.end_attribute)
def testImageMosaic(self):
# testing the mosaic creation
name = 'cea_mosaic'
data = open('test/data/mosaic/cea.zip', 'rb')
self.cat.create_imagemosaic(name, data)
# get the layer resource back
self.cat._cache.clear()
resource = self.cat.get_layer(name).resource
self.assert_(resource is not None)
# delete granule from mosaic
coverage = name
store = name
granule_id = name + '.1'
self.cat.mosaic_delete_granule(coverage, store, granule_id)
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
ft = self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
self.assertEqual(old_attribution, lyr.attribution)
def testLayerDelete(self):
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testWorkspaceDelete(self):
pass
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
pass
class UploaderTests(MapStoryTestMixin):
"""
Basic checks to make sure pages load, etc.
"""
def create_datastore(self, connection, catalog):
settings = connection.settings_dict
params = {'database': settings['NAME'],
'passwd': settings['PASSWORD'],
'namespace': 'http://www.geonode.org/',
'type': 'PostGIS',
'dbtype': 'postgis',
'host': settings['HOST'],
'user': settings['USER'],
'port': settings['PORT'],
'enabled': "True"}
store = catalog.create_datastore(settings['NAME'], workspace=self.workspace)
store.connection_parameters.update(params)
try:
catalog.save(store)
except FailedRequestError:
# assuming this is because it already exists
pass
return catalog.get_store(settings['NAME'])
def setUp(self):
if not os.path.exists(os.path.join(os.path.split(__file__)[0], 'test_ogr')):
self.skipTest('Skipping test due to missing test data.')
# These tests require geonode to be running on :80!
self.postgis = db.connections['datastore']
self.postgis_settings = self.postgis.settings_dict
self.username, self.password = self.create_user('admin', 'admin', is_superuser=True)
self.non_admin_username, self.non_admin_password = self.create_user('non_admin', 'non_admin')
self.cat = Catalog(ogc_server_settings.internal_rest, *ogc_server_settings.credentials)
self.workspace = 'geonode'
self.datastore = self.create_datastore(self.postgis, self.cat)
def tearDown(self):
"""
Clean up geoserver.
"""
self.cat.delete(self.datastore, recurse=True)
def generic_import(self, file, configuration_options=[{'index': 0}]):
f = file
filename = os.path.join(os.path.dirname(__file__), 'test_ogr', f)
res = self.import_file(filename, configuration_options=configuration_options)
layer = Layer.objects.get(name=res[0][0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
if not filename.endswith('zip'):
self.assertTrue(layer.attributes.count() >= DataSource(filename)[0].num_fields)
# make sure we have at least one dateTime attribute
self.assertTrue('xsd:dateTime' or 'xsd:date' in [n.attribute_type for n in layer.attributes.all()])
return layer
def test_box_with_year_field(self):
"""
Tests the import of test_box_with_year_field.
"""
layer = self.generic_import('boxes_with_year_field.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date(self):
"""
Tests the import of test_boxes_with_date.
"""
layer = self.generic_import('boxes_with_date.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_csv(self):
"""
Tests a CSV with WKT polygon.
"""
layer = self.generic_import('boxes_with_date.csv', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_schema_csv(self):
"""
Tests a CSV from schema download.
"""
layer = self.generic_import('schema_download.csv', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_iso_date(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_duplicate_imports(self):
"""
Tests importing the same layer twice to ensure incrementing file names is properly handled.
"""
filename = os.path.join(os.path.dirname(__file__), 'test_ogr', 'boxes_with_date_iso_date.zip')
gi = GDALImport(filename)
layers1 = gi.handle({'index': 0, 'name': 'test'})
layers2 = gi.handle({'index': 0, 'name': 'test'})
self.assertEqual(layers1[0][0], 'test')
self.assertEqual(layers2[0][0], 'test0')
def test_boxes_with_date_iso_date_zip(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.zip', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_dates_bc(self):
"""
Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('boxes_with_dates_bc.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_point_with_date(self):
"""
Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('point_with_date.geojson', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
# OGR will name geojson layers 'ogrgeojson' we rename to the path basename
self.assertTrue(layer.name.startswith('point_with_date'))
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_end_date(self):
"""
Tests the import of test_boxes_with_end_date.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('boxes_with_end_date.shp')
date_attr = filter(lambda attr: attr.attribute == 'date', layer.attributes)[0]
end_date_attr = filter(lambda attr: attr.attribute == 'enddate', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:date')
self.assertEqual(end_date_attr.attribute_type, 'xsd:date')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute, end_attribute=end_date_attr.attribute)
def test_us_states_kml(self):
"""
Tests the import of us_states_kml.
This layer has a date and an end date field that are typed correctly.
"""
# TODO: Support time in kmls.
layer = self.generic_import('us_states.kml')
def test_mojstrovka_gpx(self):
"""
Tests the import of mojstrovka.gpx.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('mojstrovka.gpx')
date_attr = filter(lambda attr: attr.attribute == 'time', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, u'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute)
def generic_time_check(self, layer, attribute=None, end_attribute=None):
"""
Convenience method to run generic tests on time layers.
"""
self.cat._cache.clear()
resource = self.cat.get_resource(layer.name, store=layer.store, workspace=self.workspace)
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual(attribute, timeInfo.attribute)
self.assertEqual(end_attribute, timeInfo.end_attribute)
def test_us_shootings_csv(self):
"""
Tests the import of US_shootings.csv.
"""
filename = 'US_shootings.csv'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename)
layer = self.generic_import(filename, configuration_options=[{'index': 0, 'convert_to_date': ['Date']}])
self.assertEqual(layer.name, 'us_shootings')
date_field = 'date'
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_field)
self.generic_time_check(layer, attribute=date_field)
def test_sitins(self):
"""
Tests the import of US_shootings.csv.
"""
filename = 'US_Civil_Rights_Sitins0.csv'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename)
layer = self.generic_import(f, configuration_options=[{'index': 0, 'convert_to_date': ['Date']}])
def get_layer_names(self, in_file):
"""
Gets layer names from a data source.
"""
ds = DataSource(in_file)
return map(lambda layer: layer.name, ds)
def test_gdal_import(self):
filename = 'point_with_date.geojson'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', filename)
self.generic_import(filename, configuration_options=[{'index': 0, 'convert_to_date': ['date']}])
def import_file(self, in_file, configuration_options=[]):
"""
Imports the file.
"""
self.assertTrue(os.path.exists(in_file))
# run ogr2ogr
gi = GDALImport(in_file)
layers = gi.handle(configuration_options=configuration_options)
return layers
@staticmethod
def createFeatureType(catalog, datastore, name):
"""
Exposes a PostGIS feature type in geoserver.
"""
headers = {"Content-type": "application/xml"}
data = "<featureType><name>{name}</name></featureType>".format(name=name)
url = datastore.href.replace(".xml", '/featuretypes.xml'.format(name=name))
headers, response = catalog.http.request(url, "POST ", data, headers)
return response
def test_create_vrt(self):
"""
Tests the create_vrt function.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'US_shootings.csv')
vrt = create_vrt(f)
vrt.seek(0)
output = vrt.read()
self.assertTrue('name="US_shootings"' in output)
self.assertTrue('<SrcDataSource>{0}</SrcDataSource>'.format(f) in output)
self.assertTrue('<GeometryField encoding="PointFromColumns" x="Longitude" y="Latitude" />'.format(f) in output)
self.assertEqual(os.path.splitext(vrt.name)[1], '.vrt')
def test_file_add_view(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
c = AdminClient()
# test login required for this view
request = c.get(reverse('uploads-new'))
self.assertEqual(request.status_code, 302)
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context['request'].path, reverse('uploads-list'))
self.assertTrue(len(response.context['object_list']) == 1)
upload = response.context['object_list'][0]
self.assertEqual(upload.user.username, 'non_admin')
self.assertEqual(upload.file_type, 'GeoJSON')
self.assertTrue(upload.uploadlayer_set.all())
self.assertEqual(upload.state, 'UPLOADED')
self.assertIsNotNone(upload.name)
uploaded_file = upload.uploadfile_set.first()
self.assertTrue(os.path.exists(uploaded_file.file.path))
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'empty_file.geojson')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('file', response.context_data['form'].errors)
def test_file_add_view_as_json(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new-json'), {'file': fp}, follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('application/json', response.get('Content-Type', ''))
content = json.loads(response.content)
self.assertIn('state', content)
self.assertIn('id', content)
def test_describe_fields(self):
"""
Tests the describe fields functionality.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'us_shootings.csv')
fields = None
with GDALInspector(f) as f:
layers = f.describe_fields()
self.assertTrue(layers[0]['name'], 'us_shootings')
self.assertEqual([n['name'] for n in layers[0]['fields']], ['Date', 'Shooter', 'Killed',
'Wounded', 'Location', 'City',
'Longitude', 'Latitude'])
self.assertEqual(layers[0]['feature_count'], 203)
def test_gdal_file_type(self):
"""
Tests the describe fields functionality.
"""
files = ((os.path.join(os.path.dirname(__file__), 'test_ogr', 'us_shootings.csv'), 'CSV'),
(os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson'), 'GeoJSON'),
(os.path.join(os.path.dirname(__file__), 'test_ogr', 'mojstrovka.gpx'), 'GPX'),
(os.path.join(os.path.dirname(__file__), 'test_ogr', 'us_states.kml'), 'KML'),
(os.path.join(os.path.dirname(__file__), 'test_ogr', 'boxes_with_year_field.shp'), 'ESRI Shapefile'),
(os.path.join(os.path.dirname(__file__), 'test_ogr', 'boxes_with_date_iso_date.zip'), 'ESRI Shapefile'),
)
for path, file_type in files:
with GDALInspector(path) as f:
self.assertEqual(f.file_type(), file_type)
def test_append(self):
"""
Tests the configuration view.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'name': 'append',
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {'users': {'test': new_user_perms,
'AnonymousUser': ["change_layer_data", "download_resourcebase",
"view_resourcebase"]}}}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
cursor = db.connections['datastore'].cursor()
cursor.execute('select count(*) from append')
self.assertEqual(1,cursor.fetchone()[0])
payload[0]['appendTo'] = 'geonode:append'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date_2.geojson')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
cursor = db.connections['datastore'].cursor()
cursor.execute('select count(*) from append')
self.assertEqual(2,cursor.fetchone()[0])
def test_trunc_append(self):
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'long_attr_name.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'name': 'append',
'convert_to_date': ['date_as_date'],
'start_date': 'date_as_date',
'configureTime': True,
'editable': True,
'permissions': {'users': {'test': new_user_perms,
'AnonymousUser': ["change_layer_data", "download_resourcebase",
"view_resourcebase"]}}}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
cursor = db.connections['datastore'].cursor()
cursor.execute('select count(*) from append')
self.assertEqual(1, cursor.fetchone()[0])
payload[0]['appendTo'] = 'geonode:append'
payload[0]['convert_to_date'] = ['date_as_da']
payload[0]['start_date'] = 'date_as_da'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'long_attr_trunc.geojson')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
cursor = db.connections['datastore'].cursor()
cursor.execute('select count(*), date from append group by date')
result = cursor.fetchone()
#ensure that the feature was added and the attribute was appended
self.assertEqual(2,result[0])
self.assertNotEqual(None,result[1])
def test_schema_append(self):
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'schema_initial.zip')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'name': 'append',
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {'users': {'test': new_user_perms,
'AnonymousUser': ["change_layer_data", "download_resourcebase",
"view_resourcebase"]}}}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
cursor = db.connections['datastore'].cursor()
cursor.execute('select count(*) from append')
self.assertEqual(1, cursor.fetchone()[0])
payload[0]['appendTo'] = 'geonode:append'
payload[0]['convert_to_date'] = ['date']
payload[0]['start_date'] = 'date'
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'schema_append.zip')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
cursor = db.connections['datastore'].cursor()
cursor.execute('select count(*), date from append group by date')
result = cursor.fetchone()
import pdb;pdb.set_trace()
#ensure that the feature was added and the attribute was appended
self.assertEqual(2,result[0])
self.assertNotEqual(None,result[1])
def test_configure_view(self):
"""
Tests the configuration view.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {'users': {'test': new_user_perms,
'AnonymousUser': ["change_layer_data", "download_resourcebase",
"view_resourcebase"]}}}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertEqual(Layer.objects.all()[0].owner.username, self.non_admin_username)
perms = layer.get_all_level_info()
user = User.objects.get(username=self.non_admin_username)
# check user permissions
for perm in [u'publish_resourcebase', u'change_resourcebase_permissions',
u'delete_resourcebase', u'change_resourcebase', u'change_resourcebase_metadata',
u'download_resourcebase', u'view_resourcebase', u'change_layer_style',
u'change_layer_data']:
self.assertIn(perm, perms['users'][user])
self.assertTrue(perms['users'][new_user])
self.assertIn('change_resourcebase_permissions', perms['users'][new_user])
self.assertIn("change_layer_data", perms['users'][User.objects.get(username='******')])
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
def test_configure_view_convert_date(self):
"""
Tests the configure view with a dataset that needs to be converted to a date.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'US_shootings.csv')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'convert_to_date': ['Date'],
'start_date': 'Date',
'configureTime': True,
'editable': True}]
response = c.get('/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 405)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 400)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertTrue(layer.attributes.filter(attribute='date'), 'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
# ensure a user who does not own the upload cannot configure an import from it.
c.logout()
c.login_as_admin()
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 404)
def test_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
admin = User.objects.get(username=self.username)
non_admin = User.objects.get(username=self.non_admin_username)
from .models import UploadFile
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'US_shootings.csv')
with open(f, 'rb') as f:
uploaded_file = SimpleUploadedFile('test_data', f.read())
admin_upload = UploadedData.objects.create(state='Admin test', user=admin)
admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file))
non_admin_upload = UploadedData.objects.create(state='Non admin test', user=non_admin)
non_admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file))
c.login_as_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 2)
response = c.get('/importer-api/data/?user__username=admin')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 1)
def test_layer_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 200)
def test_delete_from_non_admin_api(self):
"""
Ensure users can delete their data.
"""
c = AdminClient()
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def test_delete_from_admin_api(self):
"""
Ensure that administrators can delete data that isn't theirs.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
c.logout()
c.login_as_admin()
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def naming_an_import(self):
"""
Tests providing a name in the configuration options.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
name = 'point-with-a-date'
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
payload = {'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'name': name,
'editable': True}
response = c.post('/importer-api/data-layers/1/configure/', data=json.dumps(payload),
content_type='application/json')
layer = Layer.objects.all()[0]
self.assertEqual(layer.title, name.replace('-', '_'))
def test_api_import(self):
"""
Tests the import api.
"""
f = os.path.join(os.path.dirname(__file__), 'test_ogr', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
payload = {'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True}
self.assertTrue(isinstance(UploadLayer.objects.first().configuration_options, dict))
response = c.get('/importer-api/data-layers/1/')
self.assertEqual(response.status_code, 200)
response = c.post('/importer-api/data-layers/1/configure/', data=json.dumps([payload]),
content_type='application/json')
self.assertEqual(response.status_code, 200)
self.assertTrue('task' in response.content)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
self.assertTrue(UploadLayer.objects.first().task_id)
def test_valid_file_extensions(self):
"""
Test the file extension validator.
"""
for extension in IMPORTER_VALID_EXTENSIONS:
self.assertIsNone(validate_file_extension(SimpleUploadedFile('test.{0}'.format(extension), '')))
with self.assertRaises(ValidationError):
validate_file_extension(SimpleUploadedFile('test.txt', ''))
def test_no_geom(self):
"""
Test the file extension validator.
"""
with self.assertRaises(ValidationError):
validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)'))
# This should pass (geom type is unknown)
validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)'))
def test_numeric_overflow(self):
"""
Regression test for numeric field overflows in shapefiles.
# https://trac.osgeo.org/gdal/ticket/5241
"""
self.generic_import('Walmart.zip', configuration_options=[{'index': 0, 'convert_to_date': []}])
def test_outside_bounds_regression(self):
"""
Regression where layers with features outside projection bounds fail.
"""
self.generic_import('Spring_2015.zip', configuration_options=[{'index': 0 }])
resource = self.cat.get_layer('spring_2015').resource
self.assertEqual(resource.latlon_bbox, ('-180.0', '180.0', '-90.0', '90.0', 'EPSG:4326'))
def test_multipolygon_shapefile(self):
"""
Tests shapefile with multipart polygons.
"""
self.generic_import('PhoenixFirstDues.zip', configuration_options=[{'index': 0}])
def test_gwc_handler(self):
"""
Tests the GeoWebCache handler
"""
layer = self.generic_import('boxes_with_date.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
gwc = GeoWebCacheHandler(None)
gs_layer = self.cat.get_layer(layer.name)
self.assertTrue(gwc.time_enabled(gs_layer))
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertTrue('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
# Don't configure time, ensure everything still works
layer = self.generic_import('boxes_with_date_iso_date.shp', configuration_options=[{'index': 0}])
gs_layer = self.cat.get_layer(layer.name)
self.cat._cache.clear()
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertFalse('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(
host="localhost", port="5432", database="db", user="******",
passwd="password", dbtype="postgis")
self.cat.save(ds)
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(
host="localhost", port="5432", database="db", user="******",
passwd="password", dbtype="postgis")
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
def testCoverageStoreCreate(self):
ds = self.cat.create_coveragestore2("coverage_gsconfig")
ds.data_url = "file:data/mytiff.tiff"
self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats))
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
# TODO: Uploading WorldImage file no longer works???
# ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
# self.assertRaises(
# ConflictingDataError,
# lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# )
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
class UploaderTests(DjagnoOsgeoMixin):
"""
Basic checks to make sure pages load, etc.
"""
def create_datastore(self, connection, catalog):
settings = connection.settings_dict
params = {
'database': settings['NAME'],
'passwd': settings['PASSWORD'],
'namespace': 'http://www.geonode.org/',
'type': 'PostGIS',
'dbtype': 'postgis',
'host': settings['HOST'],
'user': settings['USER'],
'port': settings['PORT'],
'enabled': "True"
}
store = catalog.create_datastore(settings['NAME'],
workspace=self.workspace)
store.connection_parameters.update(params)
try:
catalog.save(store)
except FailedRequestError:
# assuming this is because it already exists
pass
return catalog.get_store(settings['NAME'])
def setUp(self):
if not os.path.exists(
os.path.join(
os.path.split(__file__)[0], '..', 'importer-test-files')):
self.skipTest('Skipping test due to missing test data.')
# These tests require geonode to be running on :80!
self.postgis = db.connections['datastore']
self.postgis_settings = self.postgis.settings_dict
self.username, self.password = self.create_user('admin',
'admin',
is_superuser=True)
self.non_admin_username, self.non_admin_password = self.create_user(
'non_admin', 'non_admin')
self.cat = Catalog(ogc_server_settings.internal_rest,
*ogc_server_settings.credentials)
if self.cat.get_workspace('geonode') == None:
self.cat.create_workspace('geonode', 'http://www.geonode.org/')
self.workspace = 'geonode'
self.datastore = self.create_datastore(self.postgis, self.cat)
def tearDown(self):
"""
Clean up geoserver.
"""
self.cat.delete(self.datastore, recurse=True)
def generic_import(self, file, configuration_options=[{'index': 0}]):
f = file
filename = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', f)
res = self.import_file(filename,
configuration_options=configuration_options)
layer_results = []
for result in res:
if result[1].get('raster'):
layerfile = result[0]
layername = os.path.splitext(os.path.basename(layerfile))[0]
layer = Layer.objects.get(name=layername)
self.assertTrue(layerfile.endswith('.tif'))
self.assertTrue(os.path.exists(layerfile))
l = gdal.OpenEx(layerfile)
self.assertTrue(l.GetDriver().ShortName, 'GTiff')
layer_results.append(layer)
else:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
if not filename.endswith('zip'):
self.assertTrue(layer.attributes.count() >= DataSource(
filename)[0].num_fields)
layer_results.append(layer)
return layer_results[0]
def generic_raster_import(self,
file,
configuration_options=[{
'index': 0
}]):
f = file
filename = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', f)
res = self.import_file(filename,
configuration_options=configuration_options)
layerfile = res[0][0]
layername = os.path.splitext(os.path.basename(layerfile))[0]
layer = Layer.objects.get(name=layername)
self.assertTrue(layerfile.endswith('.tif'))
self.assertTrue(os.path.exists(layerfile))
l = gdal.OpenEx(layerfile)
self.assertTrue(l.GetDriver().ShortName, 'GTiff')
return layer
def test_raster(self):
"""
Tests raster import
"""
layer = self.generic_raster_import('test_grid.tif',
configuration_options=[{
'index': 0
}])
def test_box_with_year_field(self):
"""
Tests the import of test_box_with_year_field.
"""
layer = self.generic_import('boxes_with_year_field.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date(self):
"""
Tests the import of test_boxes_with_date.
"""
layer = self.generic_import('boxes_with_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date'],
'start_date':
'date',
'configureTime':
True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_gpkg(self):
"""
Tests the import of test_boxes_with_date.gpkg.
"""
layer = self.generic_import('boxes_with_date.gpkg',
configuration_options=[{
'index':
0,
'convert_to_date': ['date'],
'start_date':
'date',
'configureTime':
True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_plus_raster_gpkg_by_index(self):
"""
Tests the import of multilayer vector + raster geopackage using index
"""
layer = self.generic_import('boxes_plus_raster.gpkg',
configuration_options=[
{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
},
{
'index': 1
},
{
'index': 2
},
{
'index': 3
},
{
'index': 4
},
{
'index': 5
},
{
'index': 6
},
{
'index': 7
},
])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_csv(self):
"""
Tests a CSV with WKT polygon.
"""
layer = self.generic_import('boxes_with_date.csv',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_csv_missing_features(self):
"""
Test csv that has some rows without geoms and uses ISO-8859 (Latin 4) encoding.
"""
self.generic_import('missing-features.csv',
configuration_options=[{
'index': 0
}])
def test_boxes_with_iso_date(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_duplicate_imports(self):
"""
Tests importing the same layer twice to ensure incrementing file names is properly handled.
"""
filename = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files',
'boxes_with_date_iso_date.zip')
gi = OGRImport(filename)
layers1 = gi.handle({'index': 0, 'name': 'test'})
layers2 = gi.handle({'index': 0, 'name': 'test'})
self.assertEqual(layers1[0][0], 'test')
self.assertEqual(layers2[0][0], 'test0')
def test_launder(self):
"""
Ensure the launder function works as expected.
"""
self.assertEqual(launder('tm_world_borders_simpl_0.3'),
'tm_world_borders_simpl_0_3')
self.assertEqual(launder('Testing#'), 'testing_')
self.assertEqual(launder(' '), '_')
def test_boxes_with_date_iso_date_zip(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.zip',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_dates_bc(self):
"""
Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('boxes_with_dates_bc.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_point_with_date(self):
"""
Tests the import of point_with_date.geojson
"""
layer = self.generic_import('point_with_date.geojson',
configuration_options=[{
'index':
0,
'convert_to_date': ['date']
}])
# OGR will name geojson layers 'ogrgeojson' we rename to the path basename
self.assertTrue(layer.name.startswith('point_with_date'))
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(
self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_end_date(self):
"""
Tests the import of test_boxes_with_end_date.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('boxes_with_end_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date', 'enddate'],
'start_date':
'date',
'end_date':
'enddate',
'configureTime':
True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date',
layer.attributes)[0]
end_date_attr = filter(
lambda attr: attr.attribute == 'enddate_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
self.assertEqual(end_date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
self.generic_time_check(layer,
attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
def test_us_states_kml(self):
"""
Tests the import of us_states_kml.
This layer has a date and an end date field that are typed correctly.
"""
# TODO: Support time in kmls.
layer = self.generic_import('us_states.kml',
configuration_options=[{
'index': 0
}])
def test_mojstrovka_gpx(self):
"""
Tests the import of mojstrovka.gpx.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('mojstrovka.gpx',
configuration_options=[{
'index':
0,
'convert_to_date': ['time'],
'configureTime':
True
}])
date_attr = filter(lambda attr: attr.attribute == 'time_as_date',
layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, u'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource,
attribute=date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute)
def generic_time_check(self, layer, attribute=None, end_attribute=None):
"""
Convenience method to run generic tests on time layers.
"""
self.cat._cache.clear()
resource = self.cat.get_resource(layer.name,
store=layer.store,
workspace=self.workspace)
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual(attribute, timeInfo.attribute)
self.assertEqual(end_attribute, timeInfo.end_attribute)
def test_us_shootings_csv(self):
"""
Tests the import of US_Shootings.csv.
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
filename = 'US_Shootings.csv'
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', filename)
layer = self.generic_import(filename,
configuration_options=[{
'index':
0,
'convert_to_date': ['Date']
}])
self.assertTrue(layer.name.startswith('us_shootings'))
date_field = 'date'
configure_time(self.cat.get_layer(layer.name).resource,
attribute=date_field)
self.generic_time_check(layer, attribute=date_field)
def test_sitins(self):
"""
Tests the import of US_Civil_Rights_Sitins0.csv
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
layer = self.generic_import("US_Civil_Rights_Sitins0.csv",
configuration_options=[{
'index':
0,
'convert_to_date': ['Date']
}])
def get_layer_names(self, in_file):
"""
Gets layer names from a data source.
"""
ds = DataSource(in_file)
return map(lambda layer: layer.name, ds)
def test_gdal_import(self):
filename = 'point_with_date.geojson'
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', filename)
self.generic_import(filename,
configuration_options=[{
'index': 0,
'convert_to_date': ['date']
}])
def test_wfs(self):
"""
Tests the import from a WFS Endpoint
"""
wfs = 'WFS:http://demo.boundlessgeo.com/geoserver/wfs'
gi = OGRImport(wfs)
layers = gi.handle(configuration_options=[{
'layer_name': 'og:bugsites'
}, {
'layer_name': 'topp:states'
}])
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def test_arcgisjson(self):
"""
Tests the import from a WFS Endpoint
"""
endpoint = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services/Hydrography/Watershed173811/FeatureServer/0/query?where=objectid+%3D+objectid&outfields=*&f=json'
gi = OGRImport(endpoint)
layers = gi.handle(configuration_options=[{'index': 0}])
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def import_file(self, in_file, configuration_options=[]):
"""
Imports the file.
"""
self.assertTrue(os.path.exists(in_file))
# run ogr2ogr
gi = OGRImport(in_file)
layers = gi.handle(configuration_options=configuration_options)
return layers
@staticmethod
def createFeatureType(catalog, datastore, name):
"""
Exposes a PostGIS feature type in geoserver.
"""
headers = {"Content-type": "application/xml"}
data = "<featureType><name>{name}</name></featureType>".format(
name=name)
url = datastore.href.replace(".xml",
'/featuretypes.xml'.format(name=name))
headers, response = catalog.http.request(url, "POST ", data, headers)
return response
def test_file_add_view(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
# test login required for this view
request = c.get(reverse('uploads-new'))
self.assertEqual(request.status_code, 302)
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context['request'].path,
reverse('uploads-list'))
self.assertTrue(len(response.context['object_list']) == 1)
upload = response.context['object_list'][0]
self.assertEqual(upload.user.username, 'non_admin')
self.assertEqual(upload.file_type, 'GeoJSON')
self.assertTrue(upload.uploadlayer_set.all())
self.assertEqual(upload.state, 'UPLOADED')
self.assertIsNotNone(upload.name)
uploaded_file = upload.uploadfile_set.first()
self.assertTrue(os.path.exists(uploaded_file.file.path))
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'empty_file.geojson')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('file', response.context_data['form'].errors)
def test_file_add_view_as_json(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new-json'), {'file': fp},
follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('application/json', response.get('Content-Type', ''))
content = json.loads(response.content)
self.assertIn('state', content)
self.assertIn('id', content)
def test_describe_fields(self):
"""
Tests the describe fields functionality.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'US_Shootings.csv')
with GDALInspector(f) as f:
layers = f.describe_fields()
self.assertTrue(layers[0]['layer_name'], 'us_shootings')
self.assertEqual([n['name'] for n in layers[0]['fields']], [
'Date', 'Shooter', 'Killed', 'Wounded', 'Location', 'City',
'Longitude', 'Latitude'
])
self.assertEqual(layers[0]['feature_count'], 203)
def test_gdal_file_type(self):
"""
Tests the describe fields functionality.
"""
files = [
(os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'US_Shootings.csv'), 'CSV'),
(os.path.join(os.path.dirname(__file__), '..',
'importer-test-files',
'point_with_date.geojson'), 'GeoJSON'),
(os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'mojstrovka.gpx'), 'GPX'),
(os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'us_states.kml'), 'KML'),
(os.path.join(os.path.dirname(__file__), '..',
'importer-test-files',
'boxes_with_year_field.shp'), 'ESRI Shapefile'),
(os.path.join(os.path.dirname(__file__), '..',
'importer-test-files',
'boxes_with_date_iso_date.zip'), 'ESRI Shapefile'),
]
from .models import NoDataSourceFound
try:
for path, file_type in files:
with GDALInspector(path) as f:
self.assertEqual(f.file_type(), file_type)
except NoDataSourceFound as e:
print 'No data source found in: {0}'.format(path)
raise e
def test_configure_view(self):
"""
Tests the configuration view.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
upload = response.context['object_list'][0]
payload = [{
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {
'users': {
'test':
new_user_perms,
'AnonymousUser': [
"change_layer_data", "download_resourcebase",
"view_resourcebase"
]
}
}
}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id),
data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertEqual(Layer.objects.all()[0].owner.username,
self.non_admin_username)
perms = layer.get_all_level_info()
user = User.objects.get(username=self.non_admin_username)
# check user permissions
for perm in [
u'publish_resourcebase', u'change_resourcebase_permissions',
u'delete_resourcebase', u'change_resourcebase',
u'change_resourcebase_metadata', u'download_resourcebase',
u'view_resourcebase', u'change_layer_style',
u'change_layer_data'
]:
self.assertIn(perm, perms['users'][user])
self.assertTrue(perms['users'][new_user])
self.assertIn('change_resourcebase_permissions',
perms['users'][new_user])
self.assertIn(
"change_layer_data",
perms['users'][User.objects.get(username='******')])
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
def test_configure_view_convert_date(self):
"""
Tests the configure view with a dataset that needs to be converted to a date.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'US_Shootings.csv')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
upload = response.context['object_list'][0]
payload = [{
'index': 0,
'convert_to_date': ['Date'],
'start_date': 'Date',
'configureTime': True,
'editable': True
}]
response = c.get('/importer-api/data-layers/{0}/configure/'.format(
upload.id))
self.assertEqual(response.status_code, 405)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id))
self.assertEqual(response.status_code, 400)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id),
data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertTrue(layer.attributes.filter(attribute='date'),
'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
# ensure a user who does not own the upload cannot configure an import from it.
c.logout()
c.login_as_admin()
response = c.post('/importer-api/data-layers/{0}/configure/'.format(
upload.id))
self.assertEqual(response.status_code, 404)
def test_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
admin = User.objects.get(username=self.username)
non_admin = User.objects.get(username=self.non_admin_username)
from osgeo_importer.models import UploadFile
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'US_Shootings.csv')
with open(f, 'rb') as f:
uploaded_file = SimpleUploadedFile('test_data', f.read())
admin_upload = UploadedData.objects.create(state='Admin test',
user=admin)
admin_upload.uploadfile_set.add(
UploadFile.objects.create(file=uploaded_file))
non_admin_upload = UploadedData.objects.create(
state='Non admin test', user=non_admin)
non_admin_upload.uploadfile_set.add(
UploadFile.objects.create(file=uploaded_file))
c.login_as_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 2)
response = c.get('/importer-api/data/?user__username=admin')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 1)
def test_layer_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 200)
def test_delete_from_non_admin_api(self):
"""
Ensure users can delete their data.
"""
c = AdminClient()
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def test_delete_from_admin_api(self):
"""
Ensure that administrators can delete data that isn't theirs.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
c.logout()
c.login_as_admin()
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def naming_an_import(self):
"""
Tests providing a name in the configuration options.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
name = 'point-with-a-date'
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'name': name,
'editable': True
}
response = c.post('/importer-api/data-layers/1/configure/',
data=json.dumps(payload),
content_type='application/json')
layer = Layer.objects.all()[0]
self.assertEqual(layer.title, name.replace('-', '_'))
def test_api_import(self):
"""
Tests the import api.
"""
f = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp},
follow=True)
payload = {
'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True
}
self.assertTrue(
isinstance(UploadLayer.objects.first().configuration_options,
dict))
response = c.get('/importer-api/data-layers/1/')
self.assertEqual(response.status_code, 200)
response = c.post('/importer-api/data-layers/1/configure/',
data=json.dumps([payload]),
content_type='application/json')
self.assertEqual(response.status_code, 200)
self.assertTrue('task' in response.content)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
self.assertTrue(UploadLayer.objects.first().task_id)
def test_valid_file_extensions(self):
"""
Test the file extension validator.
"""
for extension in load_handler(OSGEO_IMPORTER,
'test.txt').valid_extensions:
self.assertIsNone(
validate_file_extension(
SimpleUploadedFile('test.{0}'.format(extension), '')))
with self.assertRaises(ValidationError):
validate_file_extension(SimpleUploadedFile('test.txt', ''))
def test_no_geom(self):
"""
Test the file extension validator.
"""
with self.assertRaises(ValidationError):
validate_inspector_can_read(
SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)'))
# This should pass (geom type is unknown)
validate_inspector_can_read(
SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)'))
def test_numeric_overflow(self):
"""
Regression test for numeric field overflows in shapefiles.
# https://trac.osgeo.org/gdal/ticket/5241
"""
self.generic_import('Walmart.zip',
configuration_options=[{
"configureTime":
False,
"convert_to_date": ["W1_OPENDAT"],
"editable":
True,
"index":
0,
"name":
"Walmart",
"start_date":
"W1_OPENDAT"
}])
def test_multipolygon_shapefile(self):
"""
Tests shapefile with multipart polygons.
"""
self.generic_import('PhoenixFirstDues.zip',
configuration_options=[{
'index': 0
}])
def test_non_4326_SR(self):
"""
Tests shapefile with multipart polygons.
"""
res = self.generic_import('Istanbul.zip',
configuration_options=[{
'index': 0
}])
featuretype = self.cat.get_resource(res.name)
self.assertEqual(featuretype.projection, 'EPSG:32635')
def test_gwc_handler(self):
"""
Tests the GeoWebCache handler
"""
layer = self.generic_import('boxes_with_date.shp',
configuration_options=[{
'index':
0,
'convert_to_date': ['date'],
'start_date':
'date',
'configureTime':
True
}])
gwc = GeoWebCacheHandler(None)
gs_layer = self.cat.get_layer(layer.name)
self.assertTrue(gwc.time_enabled(gs_layer))
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertTrue('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
# Don't configure time, ensure everything still works
layer = self.generic_import('boxes_with_date_iso_date.shp',
configuration_options=[{
'index': 0
}])
gs_layer = self.cat.get_layer(layer.name)
self.cat._cache.clear()
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertFalse('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
def test_utf8(self):
"""
Tests utf8 characters in attributes
"""
filename = os.path.join(os.path.dirname(__file__), '..',
'importer-test-files', 'china_provinces.shp')
layer = self.generic_import('china_provinces.shp')
gi = OGRImport(filename)
ds, insp = gi.open_target_datastore(gi.target_store)
sql = str("select NAME_CH from %s where NAME_PY = 'An Zhou'" %
(layer.name))
res = ds.ExecuteSQL(sql)
feat = res.GetFeature(0)
self.assertEqual(feat.GetField('name_ch'), "安州")
def test_non_converted_date(self):
"""
Test converting a field as date.
"""
results = self.generic_import('TM_WORLD_BORDERS_2005.zip',
configuration_options=[{
'index':
0,
'start_date':
'Year',
'configureTime':
True
}])
layer = self.cat.get_layer(results.typename)
self.assertTrue('time' in layer.resource.metadata)
self.assertEqual('year', layer.resource.metadata['time'].attribute)
def test_fid_field(self):
"""
Regression test for preserving an FID field when target layer supports it but source does not.
"""
self.generic_import('noaa_paleoclimate.zip',
configuration_options=[{
'index': 0
}])
def test_csv_with_wkb_geometry(self):
"""
Tests problems with the CSV files with multiple geometries.
"""
files = [
'police_csv.csv', 'police_csv_nOGC.csv', 'police_csv_noLatLon.csv',
'police_csv_WKR.csv', 'police_csv_nFID.csv',
'police_csv_nOGFID.csv', 'police_csv_noWKB.csv'
]
for i in files:
self.generic_import(
i, {
"configureTime": True,
"convert_to_date": ["date_time"],
"editable": True,
"index": 0,
"name": i.lower(),
"permissions": {
"users": {
"AnonymousUser": [
"change_layer_data", "download_resourcebase",
"view_resourcebase"
]
}
},
"start_date": "date_time",
})
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'],
username=GSPARAMS['GSUSER'],
password=GSPARAMS['GSPASSWORD'])
def testAbout(self):
about_html = self.cat.about()
self.assertTrue(
'<html xmlns="http://www.w3.org/1999/xhtml"' in str(about_html))
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d+(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertTrue(pat.match('2.10.1'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
self.assertEqual("topp", self.cat.get_workspaces("topp")[-1].name)
self.assertEqual(2,
len(self.cat.get_workspaces(names=['topp', 'sde'])))
self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde')))
def testStores(self):
self.assertEqual(0, len(self.cat.get_stores("nonexistentstore")))
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(workspace=topp)))
self.assertEqual(2, len(self.cat.get_stores(workspace=sf)))
self.assertEqual(2, len(self.cat.get_stores(workspace='sf')))
self.assertEqual(
2, len(self.cat.get_stores(names='states_shapefile, sfdem')))
self.assertEqual(
2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem'])))
self.assertEqual("sfdem", self.cat.get_stores("sfdem")[-1].name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", "topp").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states",
self.cat.get_resource("states", states).name)
self.assertEqual("states",
self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title, states.abstract, states.native_bbox,
states.latlon_bbox, states.projection, states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem",
self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer":
continue
# looking for same name
ro = self.cat.get_resource(rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
# print "Updated layer: " + rd.name
count += 1
# print "Total updated layers: " + str(count)
def testLayers(self):
expected = set([
"Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies",
"tasmania_state_boundaries", "tasmania_cities", "states",
"poly_landmarks", "tiger_roads", "poi", "giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assertTrue(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assertEqual("states", states.name)
self.assertIsInstance(states.resource, ResourceInfo)
self.assertEqual(set(s.name for s in states.styles),
set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assertTrue(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assertEqual("tasmania", tas.name)
self.assertIsInstance(tas, LayerGroup)
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assertIsNotNone(self.cat.get_workspace("topp"))
tas2 = self.cat.create_layergroup("tasmania_reloaded",
tas.layers,
workspace="topp")
self.cat.save(tas2)
self.assertIsNone(self.cat.get_layergroup("tasmania_reloaded"))
self.assertIsNotNone(
self.cat.get_layergroup("tasmania_reloaded", "topp"))
tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp")
self.assertEqual("tasmania_reloaded", tas2.name)
self.assertIsInstance(tas2, LayerGroup)
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
self.assertEqual(tas2.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld",
self.cat.get_style("population").filename)
self.assertEqual("population",
self.cat.get_style("population").sld_name)
self.assertIsNone(self.cat.get_style('non-existing-style'))
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
self.assertEqual(
self.cat.get_store(workspace="best workspace ever",
name="best store ever"), None)
self.cat.get_layer("best layer ever")
self.cat.get_layergroup("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = [
'workspaces', 'test', 'datastores', u'operaci\xf3n_repo',
'featuretypes.xml'
]
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(
u, self.cat.service_url +
"/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = [
'workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'
]
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(
u, self.cat.service_url +
"/workspaces/test/datastores/test-repo/featuretypes.xml")
def _register_cascaded_layers(service, owner=None):
"""
Register layers for a cascading WMS
"""
if service.type == 'WMS' or service.type == "OWS":
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user , _password)
# Can we always assume that it is geonode?
# Should cascading layers have a separate workspace?
cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE)
if cascade_ws is None:
cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, 'cascade')
try:
store = cat.get_store(service.name,cascade_ws)
except Exception:
store = cat.create_wmsstore(service.name, cascade_ws)
wms = WebMapService(service.base_url)
layers = list(wms.contents)
count = 0
for layer in layers:
lyr = cat.get_resource(layer, store, cascade_ws)
if lyr is None:
if service.type in ["WMS","OWS"]:
resource = cat.create_wmslayer(cascade_ws, store, layer)
elif service.type == "WFS":
resource = cat.create_wfslayer(cascade_ws, store, layer)
if resource:
cascaded_layer, created = Layer.objects.get_or_create(
typename = "%s:%s" % (cascade_ws.name, resource.name),
defaults = {
"name": resource.name,
"workspace": cascade_ws.name,
"store": store.name,
"storeType": store.resource_type,
"title": resource.title or 'No title provided',
"abstract": resource.abstract or 'No abstract provided',
"owner": None,
"uuid": str(uuid.uuid4())
})
if created:
cascaded_layer.save()
if cascaded_layer is not None and cascaded_layer.bbox is None:
cascaded_layer._populate_from_gs(gs_resource=resource)
cascaded_layer.set_default_permissions()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service,
typename=cascaded_layer.name
)
service_layer.layer = cascaded_layer
service_layer.title=cascaded_layer.title,
service_layer.description=cascaded_layer.abstract,
service_layer.styles=cascaded_layer.styles
service_layer.save()
count += 1
else:
logger.error("Resource %s from store %s could not be saved as layer" % (layer, store.name))
message = "%d Layers Registered" % count
return_dict = {'status': 'ok', 'msg': message }
return HttpResponse(json.dumps(return_dict),
mimetype='application/json',
status=200)
elif service.type == 'WCS':
return HttpResponse('Not Implemented (Yet)', status=501)
else:
return HttpResponse('Invalid Service Type', status=400)
# written at 9:21, on 2017-12-20, helping function
# removing the redundant layer groups - with vacant layers
import os, sys
from geoserver.catalog import Catalog
geoserver_url = "http://172.18.77.15:8089/geoserver"
cat = Catalog(geoserver_url + "/rest", username="******", password="******")
# getting the names of all the existing workspaces
srvr_wslist = cat.get_workspaces()
for ws_obj in srvr_wslist:
ws_name = ws_obj.name
srvr_lyrgrplist = cat.get_layergroups(workspace=ws_obj)
# check if all the related layers exists in the current layer group
for grp_obj in srvr_lyrgrplist:
for mbr_lyr_name in grp_obj.layers:
if not cat.get_resource(mbr_lyr_name):
cat.delete(grp_obj)
print "Deleting group: ", grp_obj.name
break
# then removing all the layer groups with the appointed suffixes
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testAbout(self):
about_html = self.cat.about()
self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld", self.cat.get_style("population").filename)
self.assertEqual("population", self.cat.get_style("population").sld_name)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.assertRaises(FailedRequestError,
lambda: self.cat.get_resource(
workspace="best workspace ever", store="best store ever",
name="best resource ever"))
self.cat.get_layer("best layer ever")
self.cat.get_layergroup("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
class UploaderTests(DjagnoOsgeoMixin):
"""
Basic checks to make sure pages load, etc.
"""
def create_datastore(self, connection, catalog):
settings = connection.settings_dict
params = {'database': settings['NAME'],
'passwd': settings['PASSWORD'],
'namespace': 'http://www.geonode.org/',
'type': 'PostGIS',
'dbtype': 'postgis',
'host': settings['HOST'],
'user': settings['USER'],
'port': settings['PORT'],
'enabled': "True"}
store = catalog.create_datastore(settings['NAME'], workspace=self.workspace)
store.connection_parameters.update(params)
try:
catalog.save(store)
except FailedRequestError:
# assuming this is because it already exists
pass
return catalog.get_store(settings['NAME'])
def setUp(self):
if not os.path.exists(os.path.join(os.path.split(__file__)[0], '..', 'importer-test-files')):
self.skipTest('Skipping test due to missing test data.')
# These tests require geonode to be running on :80!
self.postgis = db.connections['datastore']
self.postgis_settings = self.postgis.settings_dict
self.username, self.password = self.create_user('admin', 'admin', is_superuser=True)
self.non_admin_username, self.non_admin_password = self.create_user('non_admin', 'non_admin')
self.cat = Catalog(ogc_server_settings.internal_rest, *ogc_server_settings.credentials)
if self.cat.get_workspace('geonode') == None:
self.cat.create_workspace('geonode', 'http://www.geonode.org/')
self.workspace = 'geonode'
self.datastore = self.create_datastore(self.postgis, self.cat)
def tearDown(self):
"""
Clean up geoserver.
"""
self.cat.delete(self.datastore, recurse=True)
def generic_import(self, file, configuration_options=[{'index': 0}]):
f = file
filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', f)
res = self.import_file(filename, configuration_options=configuration_options)
layer_results=[]
for result in res:
if result[1].get('raster'):
layerfile = result[0]
layername = os.path.splitext(os.path.basename(layerfile))[0]
layer = Layer.objects.get(name=layername)
self.assertTrue(layerfile.endswith('.tif'))
self.assertTrue(os.path.exists(layerfile))
l = gdal.OpenEx(layerfile)
self.assertTrue(l.GetDriver().ShortName, 'GTiff')
layer_results.append(layer)
else:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
if not filename.endswith('zip'):
self.assertTrue(layer.attributes.count() >= DataSource(filename)[0].num_fields)
layer_results.append(layer)
return layer_results[0]
def generic_raster_import(self, file, configuration_options=[{'index': 0}]):
f = file
filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', f)
res = self.import_file(filename, configuration_options=configuration_options)
layerfile = res[0][0]
layername = os.path.splitext(os.path.basename(layerfile))[0]
layer = Layer.objects.get(name=layername)
self.assertTrue(layerfile.endswith('.tif'))
self.assertTrue(os.path.exists(layerfile))
l = gdal.OpenEx(layerfile)
self.assertTrue(l.GetDriver().ShortName, 'GTiff')
return layer
def test_raster(self):
"""
Tests raster import
"""
layer = self.generic_raster_import('test_grid.tif', configuration_options=[{'index': 0}])
def test_box_with_year_field(self):
"""
Tests the import of test_box_with_year_field.
"""
layer = self.generic_import('boxes_with_year_field.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date(self):
"""
Tests the import of test_boxes_with_date.
"""
layer = self.generic_import('boxes_with_date.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_gpkg(self):
"""
Tests the import of test_boxes_with_date.gpkg.
"""
layer = self.generic_import('boxes_with_date.gpkg', configuration_options=[{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_plus_raster_gpkg_by_index(self):
"""
Tests the import of multilayer vector + raster geopackage using index
"""
layer = self.generic_import('boxes_plus_raster.gpkg', configuration_options=[{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
},
{'index':1},
{'index':2},
{'index':3},
{'index':4},
{'index':5},
{'index':6},
{'index':7},])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_date_csv(self):
"""
Tests a CSV with WKT polygon.
"""
layer = self.generic_import('boxes_with_date.csv', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_csv_missing_features(self):
"""
Test csv that has some rows without geoms and uses ISO-8859 (Latin 4) encoding.
"""
self.generic_import('missing-features.csv', configuration_options=[{'index': 0}])
def test_boxes_with_iso_date(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_duplicate_imports(self):
"""
Tests importing the same layer twice to ensure incrementing file names is properly handled.
"""
filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'boxes_with_date_iso_date.zip')
gi = OGRImport(filename)
layers1 = gi.handle({'index': 0, 'name': 'test'})
layers2 = gi.handle({'index': 0, 'name': 'test'})
self.assertEqual(layers1[0][0], 'test')
self.assertEqual(layers2[0][0], 'test0')
def test_launder(self):
"""
Ensure the launder function works as expected.
"""
self.assertEqual(launder('tm_world_borders_simpl_0.3'), 'tm_world_borders_simpl_0_3')
self.assertEqual(launder('Testing#'), 'testing_')
self.assertEqual(launder(' '), '_')
def test_boxes_with_date_iso_date_zip(self):
"""
Tests the import of test_boxes_with_iso_date.
"""
layer = self.generic_import('boxes_with_date_iso_date.zip', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_dates_bc(self):
"""
Tests the import of test_boxes_with_dates_bc.
"""
layer = self.generic_import('boxes_with_dates_bc.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_point_with_date(self):
"""
Tests the import of point_with_date.geojson
"""
layer = self.generic_import('point_with_date.geojson', configuration_options=[{'index': 0,
'convert_to_date': ['date']}])
# OGR will name geojson layers 'ogrgeojson' we rename to the path basename
self.assertTrue(layer.name.startswith('point_with_date'))
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,)
self.generic_time_check(layer, attribute=date_attr.attribute)
def test_boxes_with_end_date(self):
"""
Tests the import of test_boxes_with_end_date.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('boxes_with_end_date.shp',configuration_options=[{'index': 0,
'convert_to_date': ['date','enddate'],
'start_date': 'date',
'end_date': 'enddate',
'configureTime': True
}])
date_attr = filter(lambda attr: attr.attribute == 'date_as_date', layer.attributes)[0]
end_date_attr = filter(lambda attr: attr.attribute == 'enddate_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, 'xsd:dateTime')
self.assertEqual(end_date_attr.attribute_type, 'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute,
end_attribute=end_date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute, end_attribute=end_date_attr.attribute)
def test_us_states_kml(self):
"""
Tests the import of us_states_kml.
This layer has a date and an end date field that are typed correctly.
"""
# TODO: Support time in kmls.
layer = self.generic_import('us_states.kml',configuration_options=[{'index': 0}])
def test_mojstrovka_gpx(self):
"""
Tests the import of mojstrovka.gpx.
This layer has a date and an end date field that are typed correctly.
"""
layer = self.generic_import('mojstrovka.gpx',configuration_options=[{'index': 0,
'convert_to_date': ['time'],
'configureTime': True
}])
date_attr = filter(lambda attr: attr.attribute == 'time_as_date', layer.attributes)[0]
self.assertEqual(date_attr.attribute_type, u'xsd:dateTime')
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_attr.attribute)
self.generic_time_check(layer, attribute=date_attr.attribute)
def generic_time_check(self, layer, attribute=None, end_attribute=None):
"""
Convenience method to run generic tests on time layers.
"""
self.cat._cache.clear()
resource = self.cat.get_resource(layer.name, store=layer.store, workspace=self.workspace)
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual(attribute, timeInfo.attribute)
self.assertEqual(end_attribute, timeInfo.end_attribute)
def test_us_shootings_csv(self):
"""
Tests the import of US_Shootings.csv.
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
filename = 'US_Shootings.csv'
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', filename)
layer = self.generic_import(filename, configuration_options=[{'index': 0, 'convert_to_date': ['Date']}])
self.assertTrue(layer.name.startswith('us_shootings'))
date_field = 'date'
configure_time(self.cat.get_layer(layer.name).resource, attribute=date_field)
self.generic_time_check(layer, attribute=date_field)
def test_sitins(self):
"""
Tests the import of US_Civil_Rights_Sitins0.csv
"""
if osgeo.gdal.__version__ < '2.0.0':
self.skipTest('GDAL Version does not support open options')
layer = self.generic_import("US_Civil_Rights_Sitins0.csv", configuration_options=[{'index': 0, 'convert_to_date': ['Date']}])
def get_layer_names(self, in_file):
"""
Gets layer names from a data source.
"""
ds = DataSource(in_file)
return map(lambda layer: layer.name, ds)
def test_gdal_import(self):
filename = 'point_with_date.geojson'
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', filename)
self.generic_import(filename, configuration_options=[{'index': 0, 'convert_to_date': ['date']}])
def test_wfs(self):
"""
Tests the import from a WFS Endpoint
"""
wfs = 'WFS:http://demo.boundlessgeo.com/geoserver/wfs'
gi = OGRImport(wfs)
layers = gi.handle(configuration_options=[{'layer_name':'og:bugsites'},
{'layer_name':'topp:states'}])
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def test_arcgisjson(self):
"""
Tests the import from a WFS Endpoint
"""
endpoint = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services/Hydrography/Watershed173811/FeatureServer/0/query?where=objectid+%3D+objectid&outfields=*&f=json'
gi = OGRImport(endpoint)
layers = gi.handle(configuration_options=[{'index':0}])
for result in layers:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
def import_file(self, in_file, configuration_options=[]):
"""
Imports the file.
"""
self.assertTrue(os.path.exists(in_file))
# run ogr2ogr
gi = OGRImport(in_file)
layers = gi.handle(configuration_options=configuration_options)
return layers
@staticmethod
def createFeatureType(catalog, datastore, name):
"""
Exposes a PostGIS feature type in geoserver.
"""
headers = {"Content-type": "application/xml"}
data = "<featureType><name>{name}</name></featureType>".format(name=name)
url = datastore.href.replace(".xml", '/featuretypes.xml'.format(name=name))
headers, response = catalog.http.request(url, "POST ", data, headers)
return response
def test_file_add_view(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
# test login required for this view
request = c.get(reverse('uploads-new'))
self.assertEqual(request.status_code, 302)
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.context['request'].path, reverse('uploads-list'))
self.assertTrue(len(response.context['object_list']) == 1)
upload = response.context['object_list'][0]
self.assertEqual(upload.user.username, 'non_admin')
self.assertEqual(upload.file_type, 'GeoJSON')
self.assertTrue(upload.uploadlayer_set.all())
self.assertEqual(upload.state, 'UPLOADED')
self.assertIsNotNone(upload.name)
uploaded_file = upload.uploadfile_set.first()
self.assertTrue(os.path.exists(uploaded_file.file.path))
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'empty_file.geojson')
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('file', response.context_data['form'].errors)
def test_file_add_view_as_json(self):
"""
Tests the file_add_view.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new-json'), {'file': fp}, follow=True)
self.assertEqual(response.status_code, 200)
self.assertIn('application/json', response.get('Content-Type', ''))
content = json.loads(response.content)
self.assertIn('state', content)
self.assertIn('id', content)
def test_describe_fields(self):
"""
Tests the describe fields functionality.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'US_Shootings.csv')
with GDALInspector(f) as f:
layers = f.describe_fields()
self.assertTrue(layers[0]['layer_name'], 'us_shootings')
self.assertEqual([n['name'] for n in layers[0]['fields']], ['Date', 'Shooter', 'Killed',
'Wounded', 'Location', 'City',
'Longitude', 'Latitude'])
self.assertEqual(layers[0]['feature_count'], 203)
def test_gdal_file_type(self):
"""
Tests the describe fields functionality.
"""
files = [
(os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'US_Shootings.csv'), 'CSV'),
(os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson'), 'GeoJSON'),
(os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'mojstrovka.gpx'), 'GPX'),
(os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'us_states.kml'), 'KML'),
(os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'boxes_with_year_field.shp'), 'ESRI Shapefile'),
(os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'boxes_with_date_iso_date.zip'), 'ESRI Shapefile'),
]
from .models import NoDataSourceFound
try:
for path, file_type in files:
with GDALInspector(path) as f:
self.assertEqual(f.file_type(), file_type)
except NoDataSourceFound as e:
print 'No data source found in: {0}'.format(path)
raise e
def test_configure_view(self):
"""
Tests the configuration view.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
new_user = User.objects.create(username='******')
new_user_perms = ['change_resourcebase_permissions']
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True,
'permissions': {'users': {'test': new_user_perms,
'AnonymousUser': ["change_layer_data", "download_resourcebase",
"view_resourcebase"]}}}]
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertEqual(Layer.objects.all()[0].owner.username, self.non_admin_username)
perms = layer.get_all_level_info()
user = User.objects.get(username=self.non_admin_username)
# check user permissions
for perm in [u'publish_resourcebase', u'change_resourcebase_permissions',
u'delete_resourcebase', u'change_resourcebase', u'change_resourcebase_metadata',
u'download_resourcebase', u'view_resourcebase', u'change_layer_style',
u'change_layer_data']:
self.assertIn(perm, perms['users'][user])
self.assertTrue(perms['users'][new_user])
self.assertIn('change_resourcebase_permissions', perms['users'][new_user])
self.assertIn("change_layer_data", perms['users'][User.objects.get(username='******')])
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
def test_configure_view_convert_date(self):
"""
Tests the configure view with a dataset that needs to be converted to a date.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'US_Shootings.csv')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
upload = response.context['object_list'][0]
payload = [{'index': 0,
'convert_to_date': ['Date'],
'start_date': 'Date',
'configureTime': True,
'editable': True}]
response = c.get('/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 405)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 400)
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id), data=json.dumps(payload),
content_type='application/json')
self.assertTrue(response.status_code, 200)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
self.assertTrue(layer.attributes.filter(attribute='date'), 'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
# ensure a user who does not own the upload cannot configure an import from it.
c.logout()
c.login_as_admin()
response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload.id))
self.assertEqual(response.status_code, 404)
def test_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
admin = User.objects.get(username=self.username)
non_admin = User.objects.get(username=self.non_admin_username)
from osgeo_importer.models import UploadFile
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'US_Shootings.csv')
with open(f, 'rb') as f:
uploaded_file = SimpleUploadedFile('test_data', f.read())
admin_upload = UploadedData.objects.create(state='Admin test', user=admin)
admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file))
non_admin_upload = UploadedData.objects.create(state='Non admin test', user=non_admin)
non_admin_upload.uploadfile_set.add(UploadFile.objects.create(file=uploaded_file))
c.login_as_admin()
response = c.get('/importer-api/data/')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 2)
response = c.get('/importer-api/data/?user__username=admin')
self.assertEqual(response.status_code, 200)
body = json.loads(response.content)
self.assertEqual(len(body['objects']), 1)
def test_layer_list_api(self):
c = AdminClient()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 401)
c.login_as_non_admin()
response = c.get('/importer-api/data-layers/')
self.assertEqual(response.status_code, 200)
def test_delete_from_non_admin_api(self):
"""
Ensure users can delete their data.
"""
c = AdminClient()
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def test_delete_from_admin_api(self):
"""
Ensure that administrators can delete data that isn't theirs.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
self.assertEqual(UploadedData.objects.all().count(), 1)
c.logout()
c.login_as_admin()
id = UploadedData.objects.first().id
response = c.delete('/importer-api/data/{0}/'.format(id))
self.assertEqual(response.status_code, 204)
self.assertEqual(UploadedData.objects.all().count(), 0)
def naming_an_import(self):
"""
Tests providing a name in the configuration options.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
name = 'point-with-a-date'
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
payload = {'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'name': name,
'editable': True}
response = c.post('/importer-api/data-layers/1/configure/', data=json.dumps(payload),
content_type='application/json')
layer = Layer.objects.all()[0]
self.assertEqual(layer.title, name.replace('-', '_'))
def test_api_import(self):
"""
Tests the import api.
"""
f = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'point_with_date.geojson')
c = AdminClient()
c.login_as_non_admin()
with open(f) as fp:
response = c.post(reverse('uploads-new'), {'file': fp}, follow=True)
payload = {'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True,
'editable': True}
self.assertTrue(isinstance(UploadLayer.objects.first().configuration_options, dict))
response = c.get('/importer-api/data-layers/1/')
self.assertEqual(response.status_code, 200)
response = c.post('/importer-api/data-layers/1/configure/', data=json.dumps([payload]),
content_type='application/json')
self.assertEqual(response.status_code, 200)
self.assertTrue('task' in response.content)
layer = Layer.objects.all()[0]
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertEqual(layer.store, self.datastore.name)
self.assertEqual(layer.storeType, 'dataStore')
self.assertTrue(layer.attributes[1].attribute_type, 'xsd:dateTime')
lyr = self.cat.get_layer(layer.name)
self.assertTrue('time' in lyr.resource.metadata)
self.assertEqual(UploadLayer.objects.first().layer, layer)
self.assertTrue(UploadLayer.objects.first().task_id)
def test_valid_file_extensions(self):
"""
Test the file extension validator.
"""
for extension in load_handler(OSGEO_IMPORTER, 'test.txt').valid_extensions:
self.assertIsNone(validate_file_extension(SimpleUploadedFile('test.{0}'.format(extension), '')))
with self.assertRaises(ValidationError):
validate_file_extension(SimpleUploadedFile('test.txt', ''))
def test_no_geom(self):
"""
Test the file extension validator.
"""
with self.assertRaises(ValidationError):
validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,loc\nyes,POINT(0,0)'))
# This should pass (geom type is unknown)
validate_inspector_can_read(SimpleUploadedFile('test.csv', 'test,WKT\nyes,POINT(0,0)'))
def test_numeric_overflow(self):
"""
Regression test for numeric field overflows in shapefiles.
# https://trac.osgeo.org/gdal/ticket/5241
"""
self.generic_import('Walmart.zip', configuration_options=[{"configureTime":False,"convert_to_date":["W1_OPENDAT"],"editable":True,"index":0,"name":"Walmart","start_date":"W1_OPENDAT"}])
def test_multipolygon_shapefile(self):
"""
Tests shapefile with multipart polygons.
"""
self.generic_import('PhoenixFirstDues.zip', configuration_options=[{'index': 0}])
def test_non_4326_SR(self):
"""
Tests shapefile with multipart polygons.
"""
res = self.generic_import('Istanbul.zip', configuration_options=[{'index': 0}])
featuretype = self.cat.get_resource(res.name)
self.assertEqual(featuretype.projection, 'EPSG:32635')
def test_gwc_handler(self):
"""
Tests the GeoWebCache handler
"""
layer = self.generic_import('boxes_with_date.shp', configuration_options=[{'index': 0,
'convert_to_date': ['date'],
'start_date': 'date',
'configureTime': True
}])
gwc = GeoWebCacheHandler(None)
gs_layer = self.cat.get_layer(layer.name)
self.assertTrue(gwc.time_enabled(gs_layer))
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertTrue('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
# Don't configure time, ensure everything still works
layer = self.generic_import('boxes_with_date_iso_date.shp', configuration_options=[{'index': 0}])
gs_layer = self.cat.get_layer(layer.name)
self.cat._cache.clear()
gs_layer.fetch()
payload = self.cat.http.request(gwc.gwc_url(gs_layer))
self.assertFalse('regexParameterFilter' in payload[1])
self.assertEqual(int(payload[0]['status']), 200)
def test_utf8(self):
"""
Tests utf8 characters in attributes
"""
filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', 'china_provinces.shp')
layer = self.generic_import('china_provinces.shp')
gi = OGRImport(filename)
ds, insp = gi.open_target_datastore(gi.target_store)
sql = str("select NAME_CH from %s where NAME_PY = 'An Zhou'" % (layer.name))
res = ds.ExecuteSQL(sql)
feat = res.GetFeature(0)
self.assertEqual(feat.GetField('name_ch'), "安州")
def test_non_converted_date(self):
"""
Test converting a field as date.
"""
results = self.generic_import('TM_WORLD_BORDERS_2005.zip', configuration_options=[{'index': 0,
'start_date': 'Year',
'configureTime': True}])
layer = self.cat.get_layer(results.typename)
self.assertTrue('time' in layer.resource.metadata)
self.assertEqual('year', layer.resource.metadata['time'].attribute)
def test_fid_field(self):
"""
Regression test for preserving an FID field when target layer supports it but source does not.
"""
self.generic_import('noaa_paleoclimate.zip', configuration_options=[{'index': 0}])
def test_csv_with_wkb_geometry(self):
"""
Tests problems with the CSV files with multiple geometries.
"""
files = ['police_csv.csv', 'police_csv_nOGC.csv', 'police_csv_noLatLon.csv', 'police_csv_WKR.csv', 'police_csv_nFID.csv',
'police_csv_nOGFID.csv', 'police_csv_noWKB.csv']
for i in files:
self.generic_import(i, {"configureTime":True,"convert_to_date":["date_time"],"editable":True,"index":0,"name":i.lower(),"permissions":{"users":{"AnonymousUser":["change_layer_data","download_resourcebase","view_resourcebase"]}},"start_date":"date_time",})
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'],
username=GSPARAMS['GSUSER'],
password=GSPARAMS['GSPASSWORD'])
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assertIsNone(self.cat.get_layer('import'))
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assertIsNotNone(self.cat.get_layer('import'))
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
try:
if lyr:
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
if ds:
self.cat.delete(ds)
except Exception:
pass
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(
ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
@drop_table('import2')
def testVirtualTables(self):
ds = self.cat.create_datastore("gsconfig_import_test2")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test2")
self.cat.add_data_to_store(
ds, "import2", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
store = self.cat.get_store("gsconfig_import_test2")
geom = JDBCVirtualTableGeometry('the_geom', 'MultiPolygon', '4326')
ft_name = 'my_jdbc_vt_test'
epsg_code = 'EPSG:4326'
sql = "select * from import2 where 'STATE_NAME' = 'Illinois'"
keyColumn = None
parameters = None
jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn,
parameters)
self.cat.publish_featuretype(ft_name,
store,
epsg_code,
jdbc_virtual_table=jdbc_vt)
# DISABLED; this test works only in the very particular case
# "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR
# def testCoverageStoreCreate(self):
# ds = self.cat.create_coveragestore2("coverage_gsconfig")
# ds.data_url = "file:test/data/mytiff.tiff"
# self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(
['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEqual(set(rs.request_srs_list), srs_before,
str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(set(rs.request_srs_list), srs_after,
str(rs.request_srs_list))
# set and save response_srs_list
self.assertEqual(set(rs.response_srs_list), srs_before,
str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(set(rs.response_srs_list), srs_after,
str(rs.response_srs_list))
# set and save supported_formats
self.assertEqual(set(rs.supported_formats), formats,
str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(set(rs.supported_formats), formats_after,
str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(workspace=wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
# sanitize the layer name - validation will fail on newer geoservers
name = layer.replace(':', '_')
self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
changed_layer = added_layers[0]
self.assertEqual(True, changed_layer.advertised)
self.assertEqual(True, changed_layer.enabled)
changed_layer.advertised = False
changed_layer.enabled = False
self.cat.save(changed_layer)
self.cat._cache.clear()
changed_layer = wmsstore.get_resources()[0]
changed_layer.fetch()
self.assertEqual(False, changed_layer.advertised)
self.assertEqual(False, changed_layer.enabled)
# Testing projection and projection policy changes
changed_layer.projection = "EPSG:900913"
changed_layer.projection_policy = "REPROJECT_TO_DECLARED"
self.cat.save(changed_layer)
self.cat._cache.clear()
layer = self.cat.get_layer(changed_layer.name)
self.assertEqual(layer.resource.projection_policy,
changed_layer.projection_policy)
self.assertEqual(layer.resource.projection, changed_layer.projection)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.items():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars,
sf)
self.assertIsNotNone(self.cat.get_resource("states_test",
workspace=sf))
self.assertRaises(
ConflictingDataError, lambda: self.cat.create_featurestore(
"states_test", shapefile_plus_sidecars, sf))
self.assertRaises(
UploadError, lambda: self.cat.create_coveragestore(
"states_raster_test", shapefile_plus_sidecars, sf))
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf))
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assertIsNone(self.cat.get_layer("states_test"))
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
self.cat.create_coveragestore("Pk50095", tiffdata, sf)
self.assertIsNotNone(self.cat.get_resource("Pk50095", workspace=sf))
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf))
self.assertRaises(
UploadError, lambda: self.cat.create_featurestore(
"Pk50095_vector", tiffdata, sf))
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff))
self.cat.create_coveragestore_external_geotiff(
"Pk50095_ext", 'file:test/data/Pk50095.tif', sf)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = {
'title': 'Not the original attribution',
'width': '123',
'height': '321',
'href': 'http://www.georchestra.org',
'url':
'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg',
'type': 'image/jpeg'
}
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = next(
(s for s in lyr.styles if s.name == "pophatch"))
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# check count before tests (upload)
count = len(self.cat.get_styles())
# upload new style, verify existence
with open("test/fred.sld") as fred_sld:
self.cat.create_style("fred", fred_sld.read())
fred = self.cat.get_style("fred")
self.assertIsNotNone(fred)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
with open("test/ted.sld") as ted_sld:
self.cat.create_style("fred", ted_sld.read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assertIsNotNone(fred)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assertIsNone(self.cat.get_style("fred"))
# attempt creating new style
with open("test/fred.sld") as fred_sld:
self.cat.create_style("fred", fred_sld.read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assertIsNotNone(f)
self.assertEqual(f.name, fred.name)
# compare count after upload
self.assertEqual(count + 1, len(self.cat.get_styles()))
# attempt creating a new style without "title"
with open("test/notitle.sld") as notitle_sld:
self.cat.create_style("notitle", notitle_sld.read())
notitle = self.cat.get_style("notitle")
self.assertEqual(None, notitle.sld_title)
def testWorkspaceStyles(self):
# upload new style, verify existence
with open("test/fred.sld") as fred_sld:
self.cat.create_style("jed", fred_sld.read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assertIsNone(jed)
jed = self.cat.get_style("jed", workspace="topp")
self.assertIsNotNone(jed)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assertIsNotNone(jed)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
with open("test/ted.sld") as ted_sld:
self.cat.create_style("jed",
ted_sld.read(),
overwrite=True,
workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertIsNotNone(jed)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assertIsNone(self.cat.get_style("jed", workspace="topp"))
# attempt creating new style
with open("test/fred.sld") as fred_sld:
self.cat.create_style("jed", fred_sld.read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assertIsNotNone(f)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
with open("test/fred.sld") as fred_sld:
self.cat.create_style("ned",
fred_sld.read(),
overwrite=True,
workspace="topp")
with open("test/fred.sld") as ted_sld:
self.cat.create_style("zed",
ted_sld.read(),
overwrite=True,
workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assertIsNotNone(ned)
self.assertIsNotNone(zed)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
if lyr.default_style is not None:
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assertIsNone(ws)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assertTrue(states.enabled)
states.enabled = False
self.assertFalse(states.enabled)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assertFalse(states.enabled)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assertTrue(states.enabled)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def testImageMosaic(self):
"""
Test case for Issue #110
"""
# testing the mosaic creation
name = 'cea_mosaic'
with open('test/data/mosaic/cea.zip', 'rb') as data:
self.cat.create_imagemosaic(name, data)
# get the layer resource back
self.cat._cache.clear()
resource = self.cat.get_layer(name).resource
self.assertIsNotNone(resource)
# delete granule from mosaic
coverage = name
store = self.cat.get_store(name)
granules = self.cat.list_granules(coverage, store)
self.assertEqual(1, len(granules['features']))
granule_id = name + '.1'
self.cat.mosaic_delete_granule(coverage, store, granule_id)
granules = self.cat.list_granules(coverage, store)
self.assertEqual(0, len(granules['features']))
"""
testing external Image mosaic creation
"""
name = 'cea_mosaic_external'
path = 'test/data/mosaic/external'
self.cat.create_imagemosaic(name, path, workspace='topp')
self.cat._cache.clear()
resource = self.cat.get_layer("external").resource
self.assertIsNotNone(resource)
# add granule to mosaic
granule_path = 'test/data/mosaic/granules/cea_20150102.tif'
self.cat.add_granule(granule_path, name, workspace='topp')
granules = self.cat.list_granules("external", name, 'topp')
self.assertEqual(2, len(granules['features']))
# add external granule to mosaic
granule_path = os.path.join(
os.getcwd(), 'test/data/mosaic/granules/cea_20150103.zip')
self.cat.add_granule(granule_path, name, workspace='topp')
granules = self.cat.list_granules("external", name, 'topp')
self.assertEqual(3, len(granules['features']))
# Delete store
store = self.cat.get_store(name)
self.cat.delete(store, purge=True, recurse=True)
self.cat._cache.clear()
def testTimeDimension(self):
sf = self.cat.get_workspace("sf")
files = shapefile_and_friends(
os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date"))
self.cat.create_featurestore("boxes_with_end_date", files, sf)
get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer(
'boxes_with_end_date').resource # noqa: E501
# configure time as LIST
resource = get_resource()
timeInfo = DimensionInfo("time",
"true",
"LIST",
None,
"ISO8601",
None,
attribute="date")
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual("date", timeInfo.attribute)
self.assertEqual("ISO8601", timeInfo.units)
# disable time dimension
timeInfo = resource.metadata['time']
timeInfo.enabled = False
# since this is an xml property, it won't get written unless we modify it
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(False, timeInfo.enabled)
# configure with interval, end_attribute and enable again
timeInfo.enabled = True
timeInfo.presentation = 'DISCRETE_INTERVAL'
timeInfo.resolution = '3 days'
timeInfo.end_attribute = 'enddate'
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(True, timeInfo.enabled)
self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation)
self.assertEqual('3 days', timeInfo.resolution_str())
self.assertEqual('enddate', timeInfo.end_attribute)
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
self.gs_version = self.cat.get_short_version()
def testGSVersion(self):
version = self.cat.get_version()
pat = re.compile('\d\.\d+')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspaces(names="topp")[-1].name)
self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde'])))
self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde')))
self.assertEqual("topp", self.cat.get_workspace("topp").name)
self.assertIsNone(self.cat.get_workspace("blahblah-"))
def testStores(self):
self.assertEqual(0, len(self.cat.get_stores(names="nonexistentstore")))
topp = self.cat.get_workspaces("topp")[0]
sf = self.cat.get_workspaces("sf")[0]
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(workspaces=topp)))
self.assertEqual(2, len(self.cat.get_stores(workspaces=sf)))
self.assertEqual(2, len(self.cat.get_stores(workspaces='sf')))
self.assertEqual(2, len(self.cat.get_stores(names='states_shapefile, sfdem')))
self.assertEqual(2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem'])))
self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0].name)
self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile")[0].name)
self.assertEqual("sfdem", self.cat.get_stores(names="sfdem", workspaces=sf.name)[0].name)
self.assertEqual("sfdem", self.cat.get_stores(names="sfdem")[0].name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
self.assertIsNone(self.cat.get_store("blah+blah-"))
def testResources(self):
topp = self.cat.get_workspaces("topp")[0]
sf = self.cat.get_workspaces("sf")[0]
states = self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0]
sfdem = self.cat.get_stores(names="sfdem", workspaces=sf.name)[0]
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(2, len(self.cat.get_resources(stores=[states.name, sfdem.name])))
self.assertEqual(11, len(self.cat.get_resources(workspaces=[topp.name, sf.name])))
self.assertEqual("states", self.cat.get_resources(names="states", stores=states.name)[0].name)
self.assertEqual("states", self.cat.get_resources(names="states", workspaces=topp.name)[0].name)
self.assertEqual("states", self.cat.get_resources(names="states")[0].name)
self.assertEqual("states", self.cat.get_resource("states").name)
self.assertIsNone(self.cat.get_resource("blah+1blah-2"))
states = self.cat.get_resources(names="states")[0]
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", stores=sfdem.name)[0].name)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", workspaces=sf.name)[0].name)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem")[0].name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer":
continue
# looking for same name
ro = self.cat.get_resources(names=rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
count += 1
def testLayers(self):
if self.gs_version >= "2.13":
expected = set([
'sf:roads',
'sf:sfdem',
'nurc:mosaic',
'tiger:giant_polygon',
'sf:bugsites',
'topp:states',
'sf:streams',
'tiger:poly_landmarks',
'tiger:poi',
'topp:tasmania_water_bodies',
'tiger:tiger_roads',
'topp:tasmania_roads',
'nurc:Pk50095',
'topp:tasmania_cities',
'nurc:Img_Sample',
'sf:restricted',
'nurc:Arc_Sample',
'sf:archsites',
'topp:tasmania_state_boundaries'
])
else:
expected = set([
"Arc_Sample",
"Pk50095",
"Img_Sample",
"mosaic",
"sfdem",
"bugsites",
"restricted",
"streams",
"archsites",
"roads",
"tasmania_roads",
"tasmania_water_bodies",
"tasmania_state_boundaries",
"tasmania_cities",
"states",
"poly_landmarks",
"tiger_roads",
"poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups(names=["tasmania", "tiger-ny", "spearfish"]))
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroups(names="tasmania")[0]
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
if self.gs_version >= "2.13":
self.assertEqual(tas.layers, [
'topp:tasmania_state_boundaries',
'topp:tasmania_water_bodies',
'topp:tasmania_roads',
'topp:tasmania_cities'
], tas.layers)
else:
self.assertEqual(tas.layers, [
'tasmania_state_boundaries',
'tasmania_water_bodies',
'tasmania_roads',
'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assert_(self.cat.get_workspaces("topp")[0] is not None)
tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp")
self.cat.save(tas2)
self.assertEqual(1, len(self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")))
tas2 = self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")[0]
self.assert_("tasmania_reloaded", tas2.name)
self.assert_(isinstance(tas2, LayerGroup))
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
if self.gs_version >= "2.13":
self.assertEqual(tas2.layers, [
'topp:tasmania_state_boundaries',
'topp:tasmania_water_bodies',
'topp:tasmania_roads',
'topp:tasmania_cities'
], tas2.layers)
else:
self.assertEqual(tas2.layers, [
'tasmania_state_boundaries',
'tasmania_water_bodies',
'tasmania_roads',
'tasmania_cities'
], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_styles("population")[0].name)
self.assertEqual("popshade.sld", self.cat.get_styles("population")[0].filename)
self.assertEqual("population", self.cat.get_styles("population")[0].sld_name)
self.assertEqual("population", self.cat.get_style("population").sld_name)
self.assertIsNone(self.cat.get_style("blah+#5blah-"))
self.assertEqual(0, len(self.cat.get_styles('non-existing-style')))
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_styles("best style ever")
self.cat.get_workspaces("best workspace ever")
self.assertEqual(0, len(self.cat.get_stores(workspaces="best workspace ever", names="best store ever")))
self.cat.get_layer("best layer ever")
self.cat.get_layergroups("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assert_(self.cat.get_layer('import') is None)
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assert_(self.cat.get_layer('import') is not None)
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
try:
if lyr:
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
if ds:
self.cat.delete(ds)
except:
pass
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
@drop_table('import2')
def testVirtualTables(self):
ds = self.cat.create_datastore("gsconfig_import_test2")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test2")
self.cat.add_data_to_store(ds, "import2", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
store = self.cat.get_store("gsconfig_import_test2")
geom = JDBCVirtualTableGeometry('the_geom','MultiPolygon','4326')
ft_name = 'my_jdbc_vt_test'
epsg_code = 'EPSG:4326'
sql = "select * from import2 where 'STATE_NAME' = 'Illinois'"
keyColumn = None
parameters = None
jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters)
ft = self.cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt)
# DISABLED; this test works only in the very particular case
# "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR
# def testCoverageStoreCreate(self):
# ds = self.cat.create_coveragestore2("coverage_gsconfig")
# ds.data_url = "file:test/data/mytiff.tiff"
# self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
# sanitize the layer name - validation will fail on newer geoservers
name = layer.replace(':', '_')
new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
changed_layer = added_layers[0]
self.assertEqual(True, changed_layer.advertised)
self.assertEqual(True, changed_layer.enabled)
changed_layer.advertised = False
changed_layer.enabled = False
self.cat.save(changed_layer)
self.cat._cache.clear()
changed_layer = wmsstore.get_resources()[0]
changed_layer.fetch()
self.assertEqual(False, changed_layer.advertised)
self.assertEqual(False, changed_layer.enabled)
# Testing projection and projection policy changes
changed_layer.projection = "EPSG:900913"
changed_layer.projection_policy = "REPROJECT_TO_DECLARED"
self.cat.save(changed_layer)
self.cat._cache.clear()
layer = self.cat.get_layer(changed_layer.name)
self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy)
self.assertEqual(layer.resource.projection, changed_layer.projection)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
ft_ext = self.cat.create_coveragestore_external_geotiff("Pk50095_ext", 'file:test/data/Pk50095.tif', sf)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = { 'title': 'Not the original attribution',
'width': '123',
'height': '321',
'href': 'http://www.georchestra.org',
'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg',
'type': 'image/jpeg' }
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# check count before tests (upload)
count = len(self.cat.get_styles())
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assert_(f is not None)
self.assertEqual(f.name, fred.name)
# compare count after upload
self.assertEqual(count +1, len(self.cat.get_styles()))
# attempt creating a new style without "title"
self.cat.create_style("notitle", open("test/notitle.sld").read())
notitle = self.cat.get_style("notitle")
self.assertEqual(None, notitle.sld_title)
def testWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assert_(jed is None)
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assert_(self.cat.get_style("jed", workspace="topp") is None)
# attempt creating new style
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assert_(f is not None)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp")
self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assert_(ned is not None)
self.assert_(zed is not None)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def testImageMosaic(self):
"""
Test case for Issue #110
"""
# testing the mosaic creation
name = 'cea_mosaic'
data = open('test/data/mosaic/cea.zip', 'rb')
self.cat.create_imagemosaic(name, data)
# get the layer resource back
self.cat._cache.clear()
resource = self.cat.get_layer(name).resource
self.assert_(resource is not None)
# delete granule from mosaic
coverage = name
store = self.cat.get_store(name)
granule_id = name + '.1'
self.cat.mosaic_delete_granule(coverage, store, granule_id)
def testTimeDimension(self):
sf = self.cat.get_workspace("sf")
files = shapefile_and_friends(os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date"))
self.cat.create_featurestore("boxes_with_end_date", files, sf)
get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer('boxes_with_end_date').resource
# configure time as LIST
resource = get_resource()
timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date")
resource.metadata = {'time':timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual("date", timeInfo.attribute)
self.assertEqual("ISO8601", timeInfo.units)
# disable time dimension
timeInfo = resource.metadata['time']
timeInfo.enabled = False
# since this is an xml property, it won't get written unless we modify it
resource.metadata = {'time' : timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(False, timeInfo.enabled)
# configure with interval, end_attribute and enable again
timeInfo.enabled = True
timeInfo.presentation = 'DISCRETE_INTERVAL'
timeInfo.resolution = '3 days'
timeInfo.end_attribute = 'enddate'
resource.metadata = {'time' : timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(True, timeInfo.enabled)
self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation)
self.assertEqual('3 days', timeInfo.resolution_str())
self.assertEqual('enddate', timeInfo.end_attribute)
def __init__(self, gs_axx, dico_gs, tipo, txt=''):
"""Use OGR functions to extract basic informations about geoserver.
gs_axx = tuple like {url of a geoserver, user, password)
dico_gs = dictionary to store
tipo = format
text = dictionary of text in the selected language
"""
# connection
cat = Catalog(gs_axx[0],
gs_axx[1],
gs_axx[2],
disable_ssl_certificate_validation=gs_axx[3])
# print(dir(cat))
# -- WORKSPACES -------------------------------------------------------
workspaces = cat.get_workspaces()
for wk in workspaces:
# print(wk.name, wk.enabled, wk.resource_type, wk.wmsstore_url)
dico_gs[wk.name] = wk.href, {}
# print(dir(wk))
# -- STORES -----------------------------------------------------------
# stores = cat.get_stores()
# for st in stores:
# # print(st.name, st.enabled, st.href, st.resource_type)
# if hasattr(st, 'url'):
# url = st.url
# elif hasattr(st, 'resource_url'):
# url = st.resource_url
# else:
# print(dir(st))
# dico_gs.get(st.workspace.name)[1][st.name] = {"ds_type": st.type,
# "ds_url": url}
# print(dir(st))
# print(st.url)
# print(st.workspace)
# print(dir(st.workspace))
# print(st.workspace.name)
# -- LAYERS -----------------------------------------------------------
# resources_target = cat.get_resources(workspace='ayants-droits')
layers = cat.get_layers()
logging.info("{} layers found".format(len(layers)))
dico_layers = OrderedDict()
for layer in layers:
# print(layer.resource_type)
lyr_title = layer.resource.title
lyr_name = layer.name
lyr_wkspace = layer.resource._workspace.name
if type(layer.resource) is Coverage:
lyr_type = "coverage"
elif type(layer.resource) is FeatureType:
lyr_type = "vector"
else:
lyr_type = type(layer.resource)
# a log handshake
logging.info("{} | {} | {} | {}".format(layers.index(layer),
lyr_type, lyr_name,
lyr_title))
# # METADATA LINKS #
# download link
md_link_dl = "{0}/geoserver/{1}/ows?request=GetFeature"\
"&service=WFS&typeName={1}%3A{2}&version=2.0.0"\
"&outputFormat=SHAPE-ZIP"\
.format(url_base,
lyr_wkspace,
lyr_name)
# mapfish links
md_link_mapfish_wms = "{0}/mapfishapp/?layername={1}"\
"&owstype=WMSLayer&owsurl={0}/"\
"geoserver/{2}/ows"\
.format(url_base,
lyr_name,
lyr_wkspace)
md_link_mapfish_wfs = "{0}/mapfishapp/?layername={1}"\
"&owstype=WFSLayer&owsurl={0}/"\
"geoserver/{2}/ows"\
.format(url_base,
lyr_name,
lyr_wkspace)
md_link_mapfish_wcs = "{0}/mapfishapp/?cache=PreferNetwork"\
"&crs=EPSG:2154&format=GeoTIFF"\
"&identifier={1}:{2}"\
"&url={0}/geoserver/ows?"\
.format(url_base,
lyr_wkspace,
lyr_name)
# OC links
md_link_oc_wms = "{0}/geoserver/{1}/wms?layers={1}:{2}"\
.format(url_base,
lyr_wkspace,
lyr_name)
md_link_oc_wfs = "{0}/geoserver/{1}/ows?typeName={1}:{2}"\
.format(url_base,
lyr_wkspace,
lyr_name)
md_link_oc_wcs = "{0}/geoserver/{1}/ows?typeName={1}:{2}"\
.format(url_base,
lyr_wkspace,
lyr_name)
# CSW Querier links
md_link_csw_wms = "{0}/geoserver/ows?service=wms&version=1.3.0"\
"&request=GetCapabilities".format(url_base)
md_link_csw_wfs = "{0}/geoserver/ows?service=wfs&version=2.0.0"\
"&request=GetCapabilities".format(url_base)
# # # SERVICE LINKS #
# GeoServer Edit links
gs_link_edit = "{}/geoserver/web/?wicket:bookmarkablePage="\
":org.geoserver.web.data.resource."\
"ResourceConfigurationPage"\
"&name={}"\
"&wsName={}".format(url_base,
lyr_wkspace,
lyr_name)
# Metadata links (service => metadata)
if is_uuid(dict_match_gs_md.get(lyr_name)):
# HTML metadata
md_uuid_pure = dict_match_gs_md.get(lyr_name)
srv_link_html = "{}/portail/geocatalogue?uuid={}"\
.format(url_base, md_uuid_pure)
# XML metadata
md_uuid_formatted = "{}-{}-{}-{}-{}".format(
md_uuid_pure[:8], md_uuid_pure[8:12], md_uuid_pure[12:16],
md_uuid_pure[16:20], md_uuid_pure[20:])
srv_link_xml = "http://services.api.isogeo.com/ows/s/"\
"{1}/{2}?"\
"service=CSW&version=2.0.2&request=GetRecordById"\
"&id=urn:isogeo:metadata:uuid:{0}&"\
"elementsetname=full&outputSchema="\
"http://www.isotc211.org/2005/gmd"\
.format(md_uuid_formatted,
csw_share_id,
csw_share_token)
# add to GeoServer layer
rzourc = cat.get_resource(lyr_name,
store=layer.resource._store.name)
rzourc.metadata_links = [
('text/html', 'ISO19115:2003', srv_link_html),
('text/xml', 'ISO19115:2003', srv_link_xml),
('text/html', 'TC211', srv_link_html),
('text/xml', 'TC211', srv_link_xml)
]
# rzourc.metadata_links.append(('text/html', 'other', 'hohoho'))
cat.save(rzourc)
else:
logging.info("Service without metadata: {} ({})".format(
lyr_name, dict_match_gs_md.get(lyr_name)))
pass
dico_layers[layer.name] = {
"title": lyr_title,
"workspace": lyr_wkspace,
"store_name": layer.resource._store.name,
"store_type": layer.resource._store.type,
"lyr_type": lyr_type,
"md_link_dl": md_link_dl,
"md_link_mapfish": md_link_mapfish_wms,
"md_link_mapfish_wms": md_link_mapfish_wms,
"md_link_mapfish_wfs": md_link_mapfish_wfs,
"md_link_mapfish_wcs": md_link_mapfish_wcs,
"md_link_oc_wms": md_link_oc_wms,
"md_link_oc_wfs": md_link_oc_wfs,
"md_link_oc_wcs": md_link_oc_wcs,
"md_link_csw_wms": md_link_csw_wms,
"md_link_csw_wfs": md_link_csw_wfs,
"gs_link_edit": gs_link_edit,
"srv_link_html": srv_link_html,
"srv_link_xml": srv_link_xml,
"md_id_matching": md_uuid_pure
}
# mem clean up
# del dico_layer
# print(dico_gs.get(layer.resource._workspace.name)[1][layer.resource._store.name])
# print(dir(layer.resource))
# print(dir(layer.resource.writers))
# print(dir(layer.resource.writers.get("metadataLinks").func_dict))
# print(layer.resource.metadata)
# print(layer.resource.metadata_links)
dico_gs["layers"] = dico_layers
def geoserver_pre_save(instance, sender, **kwargs):
"""Send information to geoserver.
The attributes sent include:
* Title
* Abstract
* Name
* Keywords
* Metadata Links,
* Point of Contact name and url
"""
url = ogc_server_settings.rest
try:
gs_catalog = Catalog(url, _user, _password)
gs_resource = gs_catalog.get_resource(instance.name)
except (EnvironmentError, FailedRequestError) as e:
gs_resource = None
msg = ('Could not connect to geoserver at "%s"'
'to save information for layer "%s"' % (
ogc_server_settings.LOCATION, instance.name.encode('utf-8'))
)
logger.warn(msg, e)
# If geoserver is not online, there is no need to continue
return
# If there is no resource returned it could mean one of two things:
# a) There is a synchronization problem in geoserver
# b) The unit tests are running and another geoserver is running in the
# background.
# For both cases it is sensible to stop processing the layer
if gs_resource is None:
logger.warn('Could not get geoserver resource for %s' % instance)
return
gs_resource.title = instance.title
gs_resource.abstract = instance.abstract
gs_resource.name= instance.name
# Get metadata links
metadata_links = []
for link in instance.link_set.metadata():
metadata_links.append((link.name, link.mime, link.url))
gs_resource.metadata_links = metadata_links
#gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings,"BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_resource)
gs_layer = gs_catalog.get_layer(instance.name)
if instance.poc and instance.poc.user:
gs_layer.attribution = str(instance.poc.user)
profile = Profile.objects.get(user=instance.poc.user)
gs_layer.attribution_link = settings.SITEURL[:-1] + profile.get_absolute_url()
#gs_layer should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings,"BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_layer)
"""Get information from geoserver.
The attributes retrieved include:
* Bounding Box
* SRID
* Download links (WMS, WCS or WFS and KML)
* Styles (SLD)
"""
gs_resource = gs_catalog.get_resource(instance.name)
bbox = gs_resource.latlon_bbox
#FIXME(Ariel): Correct srid setting below
#self.srid = gs_resource.src
# Set bounding box values
instance.bbox_x0 = bbox[0]
instance.bbox_x1 = bbox[1]
instance.bbox_y0 = bbox[2]
instance.bbox_y1 = bbox[3]
instance.update_thumbnail(save=False)
def geoserver_pre_save(instance, sender, **kwargs):
"""Send information to geoserver.
The attributes sent include:
* Title
* Abstract
* Name
* Keywords
* Metadata Links,
* Point of Contact name and url
"""
url = ogc_server_settings.internal_rest
try:
gs_catalog = Catalog(url, _user, _password)
gs_resource = gs_catalog.get_resource(instance.name)
except (EnvironmentError, FailedRequestError) as e:
gs_resource = None
msg = ('Could not connect to geoserver at "%s"'
'to save information for layer "%s"' % (
ogc_server_settings.LOCATION, instance.name.encode('utf-8'))
)
logger.warn(msg, e)
# If geoserver is not online, there is no need to continue
return
# If there is no resource returned it could mean one of two things:
# a) There is a synchronization problem in geoserver
# b) The unit tests are running and another geoserver is running in the
# background.
# For both cases it is sensible to stop processing the layer
if gs_resource is None:
logger.warn('Could not get geoserver resource for %s' % instance)
return
gs_resource.title = instance.title
gs_resource.abstract = instance.abstract
gs_resource.name= instance.name
# Get metadata links
metadata_links = []
for link in instance.link_set.metadata():
metadata_links.append((link.name, link.mime, link.url))
gs_resource.metadata_links = metadata_links
#gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings,"BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_resource)
gs_layer = gs_catalog.get_layer(instance.name)
if instance.poc and instance.poc.user:
gs_layer.attribution = str(instance.poc.user)
profile = Profile.objects.get(user=instance.poc.user)
gs_layer.attribution_link = settings.SITEURL[:-1] + profile.get_absolute_url()
#gs_layer should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings,"BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_layer)
"""Get information from geoserver.
The attributes retrieved include:
* Bounding Box
* SRID
* Download links (WMS, WCS or WFS and KML)
* Styles (SLD)
"""
gs_resource = gs_catalog.get_resource(instance.name)
bbox = gs_resource.latlon_bbox
#FIXME(Ariel): Correct srid setting below
#self.srid = gs_resource.src
# Set bounding box values
instance.bbox_x0 = bbox[0]
instance.bbox_x1 = bbox[1]
instance.bbox_y0 = bbox[2]
instance.bbox_y1 = bbox[3]
instance.update_thumbnail(save=False)
def _register_cascaded_layers(service, owner=None):
"""
Register layers for a cascading WMS
"""
if service.type == "WMS" or service.type == "OWS":
cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password)
# Can we always assume that it is geonode?
# Should cascading layers have a separate workspace?
cascade_ws = cat.get_workspace(service.name)
if cascade_ws is None:
cascade_ws = cat.create_workspace(service.name, "cascade")
try:
store = cat.get_store(service.name, cascade_ws)
except Exception:
store = cat.create_wmsstore(service.name, cascade_ws)
cat.save(store)
wms = WebMapService(service.base_url)
layers = list(wms.contents)
count = 0
for layer in layers:
lyr = cat.get_resource(layer, store, cascade_ws)
if lyr is None:
if service.type in ["WMS", "OWS"]:
resource = cat.create_wmslayer(cascade_ws, store, layer)
elif service.type == "WFS":
resource = cat.create_wfslayer(cascade_ws, store, layer)
if resource:
bbox = resource.latlon_bbox
cascaded_layer, created = Layer.objects.get_or_create(
typename="%s:%s" % (cascade_ws.name, resource.name),
service=service,
defaults={
"name": resource.name,
"workspace": cascade_ws.name,
"store": store.name,
"storeType": store.resource_type,
"title": resource.title or "No title provided",
"abstract": resource.abstract or "No abstract provided",
"owner": None,
"uuid": str(uuid.uuid4()),
"bbox_x0": bbox[0],
"bbox_x1": bbox[1],
"bbox_y0": bbox[2],
"bbox_y1": bbox[3],
},
)
if created:
cascaded_layer.save()
if cascaded_layer is not None and cascaded_layer.bbox is None:
cascaded_layer._populate_from_gs(gs_resource=resource)
cascaded_layer.set_default_permissions()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service, typename=cascaded_layer.name
)
service_layer.layer = cascaded_layer
service_layer.title = (cascaded_layer.title,)
service_layer.description = (cascaded_layer.abstract,)
service_layer.styles = cascaded_layer.styles
service_layer.save()
count += 1
else:
logger.error("Resource %s from store %s could not be saved as layer" % (layer, store.name))
message = "%d Layers Registered" % count
return_dict = {"status": "ok", "msg": message}
return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200)
elif service.type == "WCS":
return HttpResponse("Not Implemented (Yet)", status=501)
else:
return HttpResponse("Invalid Service Type", status=400)
def geoserver_post_save(instance, sender, **kwargs):
"""Save keywords to GeoServer
The way keywords are implemented requires the layer
to be saved to the database before accessing them.
"""
url = ogc_server_settings.internal_rest
try:
gs_catalog = Catalog(url, _user, _password)
gs_resource = gs_catalog.get_resource(instance.name)
except (FailedRequestError, EnvironmentError) as e:
msg = ('Could not connect to geoserver at "%s"'
'to save information for layer "%s"' % (
ogc_server_settings.LOCATION, instance.name.encode('utf-8'))
)
logger.warn(msg, e)
# If geoserver is not online, there is no need to continue
return
# If there is no resource returned it could mean one of two things:
# a) There is a synchronization problem in geoserver
# b) The unit tests are running and another geoserver is running in the
# background.
# For both cases it is sensible to stop processing the layer
if gs_resource is None:
logger.warn('Could not get geoserver resource for %s' % instance)
return
gs_resource.keywords = instance.keyword_list()
#gs_resource should only be called if ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings,"BACKEND_WRITE_ENABLED", True):
gs_catalog.save(gs_resource)
bbox = gs_resource.latlon_bbox
dx = float(bbox[1]) - float(bbox[0])
dy = float(bbox[3]) - float(bbox[2])
dataAspect = 1 if dy == 0 else dx / dy
height = 550
width = int(height * dataAspect)
# Set download links for WMS, WCS or WFS and KML
links = wms_links(ogc_server_settings.public_url + 'wms?',
instance.typename.encode('utf-8'), instance.bbox_string,
instance.srid, height, width)
for ext, name, mime, wms_url in links:
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
name=ugettext(name),
defaults=dict(
extension=ext,
url=wms_url,
mime=mime,
link_type='image',
)
)
if instance.storeType == "dataStore":
links = wfs_links(ogc_server_settings.public_url + 'wfs?', instance.typename.encode('utf-8'))
for ext, name, mime, wfs_url in links:
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=wfs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
url=wfs_url,
link_type='data',
)
)
elif instance.storeType == 'coverageStore':
#FIXME(Ariel): This works for public layers, does it work for restricted too?
# would those end up with no geotiff links, like, forever?
permissions = {}
permissions['anonymous'] = instance.get_gen_level(ANONYMOUS_USERS)
permissions['authenticated'] = instance.get_gen_level(AUTHENTICATED_USERS)
instance.set_gen_level(ANONYMOUS_USERS,'layer_readonly')
links = wcs_links(ogc_server_settings.public_url + 'wcs?', instance.typename.encode('utf-8'),
bbox=instance.bbox[:-1], crs=instance.bbox[-1], height=height, width=width)
for ext, name, mime, wcs_url in links:
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=wcs_url,
defaults=dict(
extension=ext,
name=name,
mime=mime,
link_type='data',
)
)
instance.set_gen_level(ANONYMOUS_USERS,permissions['anonymous'])
instance.set_gen_level(AUTHENTICATED_USERS,permissions['authenticated'])
kml_reflector_link_download = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode({
'layers': instance.typename.encode('utf-8'),
'mode': "download"
})
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=kml_reflector_link_download,
defaults=dict(
extension='kml',
name=_("KML"),
mime='text/xml',
link_type='data',
)
)
kml_reflector_link_view = ogc_server_settings.public_url + "wms/kml?" + urllib.urlencode({
'layers': instance.typename.encode('utf-8'),
'mode': "refresh"
})
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=kml_reflector_link_view,
defaults=dict(
extension='kml',
name=_("View in Google Earth"),
mime='text/xml',
link_type='data',
)
)
tile_url = ('%sgwc/service/gmaps?' % ogc_server_settings.public_url +
'layers=%s' % instance.typename.encode('utf-8') +
'&zoom={z}&x={x}&y={y}' +
'&format=image/png8'
)
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=tile_url,
defaults=dict(
extension='tiles',
name=_("Tiles"),
mime='image/png',
link_type='image',
)
)
html_link_url = '%s%s' % (settings.SITEURL[:-1], instance.get_absolute_url())
Link.objects.get_or_create(resource= instance.resourcebase_ptr,
url=html_link_url,
defaults=dict(
extension='html',
name=instance.typename,
mime='text/html',
link_type='html',
)
)
#remove links that belong to and old address
for link in instance.link_set.all():
if not urlparse(settings.SITEURL).hostname == urlparse(link.url).hostname and not \
urlparse(ogc_server_settings.public_url).hostname == urlparse(link.url).hostname:
link.delete()
#Save layer attributes
set_attributes(instance)
#Save layer styles
set_styles(instance, gs_catalog)
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testAbout(self):
about_html = self.cat.about()
self.assertTrue(
'<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states",
self.cat.get_resource("states", states).name)
self.assertEqual("states",
self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title, states.abstract, states.native_bbox,
states.latlon_bbox, states.projection, states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem",
self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set([
"Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies",
"tasmania_state_boundaries", "tasmania_cities", "states",
"poly_landmarks", "tiger_roads", "poi", "giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles),
set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld",
self.cat.get_style("population").filename)
self.assertEqual("population",
self.cat.get_style("population").sld_name)
self.assert_(self.cat.get_style('non-existing-style') is None)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
try:
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.fail('expected exception')
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever',
fre.message)
try:
self.cat.get_resource(workspace="best workspace ever",
store="best store ever",
name="best resource ever")
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever',
fre.message)
cat = Catalog("http://localhost:8080/geoserver/rest")
resources = cat.get_resources()
gliStores = cat.get_stores()
iWorkspaces = cat.get_workspaces()
for loWorkspace in iWorkspaces:
print loWorkspace.name
print
for loStore in gliStores:
print loStore.name
print
iLayers = cat.get_layers()
for ilLayer in iLayers:
ilNomeLayer = ilLayer.name
print ilNomeLayer
print
for ilLayer in iLayers:
ilNomeLayer = ilLayer.name
if ilNomeLayer == 'ports_eu':
ilLayerSelezionatoRisorsa = cat.get_resource(ilNomeLayer)
print ilLayerSelezionatoRisorsa.attributes
