def remove_service(request, service_id):
"""
Delete a service, and its constituent layers.
"""
service_obj = get_object_or_404(Service, pk=service_id)
if not request.user.has_perm("maps.delete_service", obj=service_obj):
return HttpResponse(
loader.render_to_string(
"401.html",
RequestContext(request, {"error_message": _("You are not permitted to remove this service.")}),
),
status=401,
)
if request.method == "GET":
return render_to_response("services/service_remove.html", RequestContext(request, {"service": service_obj}))
elif request.method == "POST":
# Retrieve this service's workspace from the GeoServer catalog.
cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password)
workspace = cat.get_workspace(service_obj.name)
# Delete nested workspace structure from GeoServer for this service.
if workspace:
for store in cat.get_stores(workspace):
for resource in cat.get_resources(store):
for layer in cat.get_layers(resource):
cat.delete(layer)
cat.delete(resource)
cat.delete(store)
cat.delete(workspace)
# Delete service from GeoNode.
service_obj.delete()
return HttpResponseRedirect(reverse("services"))
def get_geoserver_layers(geoserver_url, geoserver_username, geoserver_password):
"""
returns an iterable of GeoServer layers
"""
geoserver_rest_url = geoserver_url + 'rest'
eprint("retrieving GeoServer layer info via {}".format(geoserver_rest_url))
cat = Catalog(geoserver_rest_url, username=geoserver_username, password=geoserver_password)
all_layers = cat.get_layers()
return all_layers
def raster_to_geoserver(geoserver_url, raster, workspace):
cat = Catalog(geoserver_url, username, password)
create_workspace(geoserver_url, username, password, workspace)
file_name = os.path.splitext(os.path.basename(raster))[0]
workspace = cat.get_workspace(workspace)
if not file_name in [l.name for l in cat.get_layers()]:
print 'Creating new layer: %s' % file_name
cat.create_coveragestore(file_name, raster, workspace)
else:
print 'Layer %s already exists' % file_name
def raster_to_geoserver(geoserver_url, raster, workspace):
cat = Catalog(geoserver_url, username, password)
create_workspace(geoserver_url, username, password, workspace)
file_name = os.path.splitext(os.path.basename(raster))[0]
workspace = cat.get_workspace(workspace)
if not file_name in [l.name for l in cat.get_layers()]:
print 'Creating new layer: %s' %file_name
cat.create_coveragestore(file_name, raster, workspace)
else:
print 'Layer %s already exists' %file_name
def shp_to_geoserver(geoserver_url, shapefile, workspace):
cat = Catalog(geoserver_url, username, password)
create_workspace(geoserver_url, username, password, workspace)
shapefile_plus_sidecars = geoserver.util.shapefile_and_friends(shapefile)
file_name = os.path.splitext(os.path.basename(shapefile))[0]
workspace = cat.get_workspace(workspace)
if not file_name in [l.name for l in cat.get_layers()]:
print 'Creating new layer: %s' %file_name
cat.create_featurestore(file_name, shapefile_plus_sidecars, workspace)
else:
print 'Layer %s already exists' %file_name
def shp_to_geoserver(geoserver_url, shapefile, workspace):
cat = Catalog(geoserver_url, username, password)
create_workspace(geoserver_url, username, password, workspace)
shapefile_plus_sidecars = geoserver.util.shapefile_and_friends(shapefile)
file_name = os.path.splitext(os.path.basename(shapefile))[0]
workspace = cat.get_workspace(workspace)
if not file_name in [l.name for l in cat.get_layers()]:
print 'Creating new layer: %s' % file_name
cat.create_featurestore(file_name, shapefile_plus_sidecars, workspace)
else:
print 'Layer %s already exists' % file_name
def __init__(self, gs_url, dico_gs, tipo, txt=''):
u""" Uses OGR functions to extract basic informations about
geographic Web Features Services.
gs_url = url of a WFS service
dico_wfs = dictionary for global informations
dico_fields = dictionary for the fields' informations
li_fieds = ordered list of fields
tipo = format
text = dictionary of text in the selected language
"""
cat = Catalog(gs_url, "####", "####")
layers = cat.get_layers()
for layer in layers:
print(layer.name)
def soilmaps(request):
# return list of soilmap layers from geoserver
geoserver_api = settings.GEOSERVER_URL + '/rest'
cat = Catalog(geoserver_api, settings.GEOSERVER_USER,
settings.GEOSERVER_PASS)
#gsdm_space = cat.get_resource(workspace='gsdm')
all_layers = cat.get_layers()
soil_maps = []
for layer in all_layers:
layer_name = layer.name
if not fnmatch.fnmatch(layer_name, 'mri*'):
soil_maps.append(layer_name)
layers_json = {'soil_maps': soil_maps}
return JsonResponse(layers_json)
def __init__(self, gs_axx, dico_gs, tipo, txt=''):
u""" Uses OGR functions to extract basic informations about
geographic Web Features Services.
gs_axx = tuple like {url of a geoserver, user, password)
dico_gs = dictionary to store
tipo = format
text = dictionary of text in the selected language
"""
# connection
cat = Catalog(gs_axx[0], gs_axx[1], gs_axx[2])
# print(dir(cat))
# workspaces
workspaces = cat.get_workspaces()
for wk in workspaces:
# print(wk.name, wk.enabled, , wk.resource_type, wk.wmsstore_url)
dico_gs[wk.name] = wk.href
# print(dir(wk))
# stores
stores = cat.get_stores()
for st in stores:
# print(st.name, st.enabled, st.href, st.resource_type)
dico_gs[st.name] = st.href
print(dir(st))
dico_stores = OrderedDict()
# layers
layers = cat.get_layers()
dico_layers = OrderedDict()
for layer in layers:
# print(layer.name, layer.enabled, layer.resource._store.name, layer.resource._workspace.name)
title = layer.resource.title
print(
(layer.resource._workspace.name + "/wms?layers={}:{};".format(
layer.resource._workspace.name, layer.name),
title.encode("utf8")))
# print(title.encode("utf8"))
# dico_layers[layer.name] = layer.enabled, layer.resource.title, layer.resource.abstract, layer.resource.keywords
# dico_stores[layer.resource._store.name] = dico_layers
# dico_gs[layer.resource._workspace] = dico_stores
print(dir(layer.resource))
def get_layers():
module_dir = os.path.dirname(__file__)
file_path = os.path.join(module_dir, 'layers.txt')
data_file = open(file_path, 'w')
geoserver_api = settings.GEOSERVER_URL + '/rest'
cat = Catalog(geoserver_api, settings.GEOSERVER_USER,
settings.GEOSERVER_PASS)
all_layers = cat.get_layers()
for layer in all_layers:
layer_name = layer.name
if fnmatch.fnmatch(layer_name, '*rangelands*'):
layer_name = layer_name.replace("rangelands:", "")
data_file.write(layer_name)
data_file.write("\n")
data_file.close()
def latest_wms(request):
"""
Return latest modis dekadal wms layer
"""
geoserver_api = settings.GEOSERVER_URL + '/rest'
cat = Catalog(geoserver_api, settings.GEOSERVER_USER,
settings.GEOSERVER_PASS)
all_layers = cat.get_layers()
dekadal = []
for layer in all_layers:
layer_name = layer.name
if fnmatch.fnmatch(layer_name, 'modis.dekadal.2020*'):
dekadal.append(layer_name)
recent_wms = {'recent_wms': max(dekadal)}
return HttpResponse(json.dumps(recent_wms),
content_type="application/json")
def __init__(self, gs_axx, dico_gs, tipo, txt=''):
u""" Uses OGR functions to extract basic informations about
geographic Web Features Services.
gs_axx = tuple like {url of a geoserver, user, password)
dico_gs = dictionary to store
tipo = format
text = dictionary of text in the selected language
"""
# connection
cat = Catalog(gs_axx[0], gs_axx[1], gs_axx[2])
# print(dir(cat))
# workspaces
workspaces = cat.get_workspaces()
for wk in workspaces:
# print(wk.name, wk.enabled, , wk.resource_type, wk.wmsstore_url)
dico_gs[wk.name] = wk.href
# print(dir(wk))
# stores
stores = cat.get_stores()
for st in stores:
# print(st.name, st.enabled, st.href, st.resource_type)
dico_gs[st.name] = st.href
print(dir(st))
dico_stores = OrderedDict()
# layers
layers = cat.get_layers()
dico_layers = OrderedDict()
for layer in layers:
# print(layer.name, layer.enabled, layer.resource._store.name, layer.resource._workspace.name)
title = layer.resource.title
print((layer.resource._workspace.name + "/wms?layers={}:{};".format(layer.resource._workspace.name, layer.name), title.encode("utf8")))
# print(title.encode("utf8"))
# dico_layers[layer.name] = layer.enabled, layer.resource.title, layer.resource.abstract, layer.resource.keywords
# dico_stores[layer.resource._store.name] = dico_layers
# dico_gs[layer.resource._workspace] = dico_stores
print(dir(layer.resource))
def geoserver_add(request):
response = {}
if request.is_ajax() and request.method == 'POST':
info = request.POST
geoserver_name = info.get('geoserver_name')
geoserver_url = info.get('geoserver_url')
geoserver_username = info.get('geoserver_username')
geoserver_password = info.get('geoserver_password')
try:
cat = Catalog(geoserver_url,
username=geoserver_username,
password=geoserver_password,
disable_ssl_certificate_validation=True)
layer_list = cat.get_layers()
if layer_list:
Session = Grace.get_persistent_store_database(
'main_db', as_sessionmaker=True)
session = Session()
geoserver = Geoserver(name=geoserver_name,
url=geoserver_url,
username=geoserver_username,
password=geoserver_password)
session.add(geoserver)
session.commit()
session.close()
response = {"data": geoserver_name, "success": "Success"}
except Exception as e:
print e
response = {
"error":
"Error processing the Geoserver URL. Please check the url,username and password."
}
return JsonResponse(response)
def remove_service(request, service_id):
"""
Delete a service, and its constituent layers.
"""
service_obj = get_object_or_404(Service, pk=service_id)
if not request.user.has_perm('maps.delete_service', obj=service_obj):
return HttpResponse(loader.render_to_string('401.html',
RequestContext(request, {
'error_message':
_("You are not permitted to remove this service.")
})), status=401)
if request.method == 'GET':
return render_to_response("services/service_remove.html", RequestContext(request, {
"service": service_obj
}))
elif request.method == 'POST':
# Retrieve this service's workspace from the GeoServer catalog.
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user, _password)
workspace = cat.get_workspace(service_obj.name)
# Delete nested workspace structure from GeoServer for this service.
if workspace:
for store in cat.get_stores(workspace):
for resource in cat.get_resources(store):
for layer in cat.get_layers(resource):
cat.delete(layer)
cat.delete(resource)
cat.delete(store)
cat.delete(workspace)
# Delete service from GeoNode.
service_obj.delete()
return HttpResponseRedirect(reverse("services"))
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testAbout(self):
about_html = self.cat.about()
self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld", self.cat.get_style("population").filename)
self.assertEqual("population", self.cat.get_style("population").sld_name)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.assertRaises(FailedRequestError,
lambda: self.cat.get_resource(
workspace="best workspace ever", store="best store ever",
name="best resource ever"))
self.cat.get_layer("best layer ever")
self.cat.get_layergroup("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
data=payload,
headers=headers)
layerurl = '%s/geoserver/rest/layers/%s' % (geoserverurl, layername)
layerheaders = {'Content-type': 'application/json'}
jsonstr = {'layer': {'defaultStyle': {'name': layername}}}
layerpayload = json.dumps(jsonstr)
layerresp = requests.put(layerurl,
auth=(user, pw),
data=layerpayload,
headers=layerheaders)
# pubilsh grouplayers
geourl = "%s/geoserver/rest/" % (geoserverurl) # the url of geoserver
geocat = Catalog(geourl, user, pw) # create a Catalog object
workspace = geocat.get_workspace(targeworkspace) # workspace name
layers = geocat.get_layers()
lys = []
for layer in layersconfig:
size = len(targeworkspace) + 1
isingeoserver = False
if (layernames.__contains__(layer)):
for geoserverlyr in layers:
if geoserverlyr.name[size:] == layer:
isingeoserver = True
lys.append(targeworkspace + ":" + layer)
break
else:
print(layer + " not in slds!!")
if not isingeoserver:
print(layer + " not in geoserver!!")
layersgroup = geocat.create_layergroup(targeworkspace, lys, lys, None,
#!/usr/bin/env python
"""
gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API.
The project is distributed under a MIT License .
"""
__author__ = "David Winslow"
__copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans"
__license__ = "MIT"
from geoserver.catalog import Catalog
style_to_check = "point"
cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver")
def has_the_style(l):
return (l.default_style.name == style_to_check
or any(s.name == style_to_check for s in l.styles))
print([l.name for l in cat.get_layers() if has_the_style(l)])
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
def testAbout(self):
about_html = self.cat.about()
self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer": continue
# looking for same name
ro = self.cat.get_resource(rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
# print "Updated layer: " + rd.name
count += 1
# print "Total updated layers: " + str(count)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assert_(self.cat.get_workspace("topp") is not None)
tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp")
self.cat.save(tas2)
self.assert_(self.cat.get_layergroup("tasmania_reloaded") is None)
self.assert_(self.cat.get_layergroup("tasmania_reloaded", "topp") is not None)
tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp")
self.assert_("tasmania_reloaded", tas2.name)
self.assert_(isinstance(tas2, LayerGroup))
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
self.assertEqual(tas2.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld", self.cat.get_style("population").filename)
self.assertEqual("population", self.cat.get_style("population").sld_name)
self.assert_(self.cat.get_style('non-existing-style') is None)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
try:
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.fail('expected exception')
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever', fre.message)
try:
self.cat.get_resource(workspace="best workspace ever",
store="best store ever",
name="best resource ever")
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever', fre.message)
#!/usr/bin/env python
from geoserver.catalog import Catalog
style_to_check = "point"
cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver")
def has_the_style(l):
return (l.default_style.name == style_to_check or
any(s.name == style_to_check for s in l.styles))
print [l.name for l in cat.get_layers() if has_the_style(l)]
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'],
username=GSPARAMS['GSUSER'],
password=GSPARAMS['GSPASSWORD'])
def testAbout(self):
about_html = self.cat.about()
self.assertTrue(
'<html xmlns="http://www.w3.org/1999/xhtml"' in str(about_html))
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d+(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertTrue(pat.match('2.10.1'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
self.assertEqual("topp", self.cat.get_workspaces("topp")[-1].name)
self.assertEqual(2,
len(self.cat.get_workspaces(names=['topp', 'sde'])))
self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde')))
def testStores(self):
self.assertEqual(0, len(self.cat.get_stores("nonexistentstore")))
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(workspace=topp)))
self.assertEqual(2, len(self.cat.get_stores(workspace=sf)))
self.assertEqual(2, len(self.cat.get_stores(workspace='sf')))
self.assertEqual(
2, len(self.cat.get_stores(names='states_shapefile, sfdem')))
self.assertEqual(
2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem'])))
self.assertEqual("sfdem", self.cat.get_stores("sfdem")[-1].name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", "topp").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states",
self.cat.get_resource("states", states).name)
self.assertEqual("states",
self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title, states.abstract, states.native_bbox,
states.latlon_bbox, states.projection, states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem",
self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer":
continue
# looking for same name
ro = self.cat.get_resource(rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
# print "Updated layer: " + rd.name
count += 1
# print "Total updated layers: " + str(count)
def testLayers(self):
expected = set([
"Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies",
"tasmania_state_boundaries", "tasmania_cities", "states",
"poly_landmarks", "tiger_roads", "poi", "giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assertTrue(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assertEqual("states", states.name)
self.assertIsInstance(states.resource, ResourceInfo)
self.assertEqual(set(s.name for s in states.styles),
set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assertTrue(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assertEqual("tasmania", tas.name)
self.assertIsInstance(tas, LayerGroup)
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assertIsNotNone(self.cat.get_workspace("topp"))
tas2 = self.cat.create_layergroup("tasmania_reloaded",
tas.layers,
workspace="topp")
self.cat.save(tas2)
self.assertIsNone(self.cat.get_layergroup("tasmania_reloaded"))
self.assertIsNotNone(
self.cat.get_layergroup("tasmania_reloaded", "topp"))
tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp")
self.assertEqual("tasmania_reloaded", tas2.name)
self.assertIsInstance(tas2, LayerGroup)
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
self.assertEqual(tas2.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld",
self.cat.get_style("population").filename)
self.assertEqual("population",
self.cat.get_style("population").sld_name)
self.assertIsNone(self.cat.get_style('non-existing-style'))
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
self.assertEqual(
self.cat.get_store(workspace="best workspace ever",
name="best store ever"), None)
self.cat.get_layer("best layer ever")
self.cat.get_layergroup("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = [
'workspaces', 'test', 'datastores', u'operaci\xf3n_repo',
'featuretypes.xml'
]
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(
u, self.cat.service_url +
"/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = [
'workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'
]
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(
u, self.cat.service_url +
"/workspaces/test/datastores/test-repo/featuretypes.xml")
#!/usr/bin/python
# Objeto: contar los workspaces, con la libreria gsconfig
#
# Parametros: ninguno
from geoserver.catalog import Catalog
import sys
user = '******'
password = '******'
cat = Catalog("http://www.geo.gob.bo/geoserver/rest", user, password)
# Recuperar los datos
all_layers = cat.get_layers()
# Recuperar estadisticas sobre las capas
sin_metadata = 0
nb_capas = len(all_layers)
print "Capas sin metadatos:"
print ""
sys.stdout.flush()
for i,layer in enumerate(all_layers):
try:
if (not layer.resource.metadata_links):
sin_metadata += (not layer.resource.metadata_links)
print "%d/%d - %s - %s\n" % (i,nb_capas,layer.resource.workspace,layer.name),
sys.stdout.flush()
except:
pass
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testAbout(self):
about_html = self.cat.about()
self.assertTrue(
'<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states",
self.cat.get_resource("states", states).name)
self.assertEqual("states",
self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title, states.abstract, states.native_bbox,
states.latlon_bbox, states.projection, states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem",
self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set([
"Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies",
"tasmania_state_boundaries", "tasmania_cities", "states",
"poly_landmarks", "tiger_roads", "poi", "giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles),
set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld",
self.cat.get_style("population").filename)
self.assertEqual("population",
self.cat.get_style("population").sld_name)
self.assert_(self.cat.get_style('non-existing-style') is None)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
try:
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.fail('expected exception')
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever',
fre.message)
try:
self.cat.get_resource(workspace="best workspace ever",
store="best store ever",
name="best resource ever")
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever',
fre.message)
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
self.gs_version = self.cat.get_short_version()
def testGSVersion(self):
version = self.cat.get_version()
pat = re.compile('\d\.\d+')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspaces(names="topp")[-1].name)
self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde'])))
self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde')))
self.assertEqual("topp", self.cat.get_workspace("topp").name)
self.assertIsNone(self.cat.get_workspace("blahblah-"))
def testStores(self):
self.assertEqual(0, len(self.cat.get_stores(names="nonexistentstore")))
topp = self.cat.get_workspaces("topp")[0]
sf = self.cat.get_workspaces("sf")[0]
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(workspaces=topp)))
self.assertEqual(2, len(self.cat.get_stores(workspaces=sf)))
self.assertEqual(2, len(self.cat.get_stores(workspaces='sf')))
self.assertEqual(2, len(self.cat.get_stores(names='states_shapefile, sfdem')))
self.assertEqual(2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem'])))
self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0].name)
self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile")[0].name)
self.assertEqual("sfdem", self.cat.get_stores(names="sfdem", workspaces=sf.name)[0].name)
self.assertEqual("sfdem", self.cat.get_stores(names="sfdem")[0].name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
self.assertIsNone(self.cat.get_store("blah+blah-"))
def testResources(self):
topp = self.cat.get_workspaces("topp")[0]
sf = self.cat.get_workspaces("sf")[0]
states = self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0]
sfdem = self.cat.get_stores(names="sfdem", workspaces=sf.name)[0]
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(2, len(self.cat.get_resources(stores=[states.name, sfdem.name])))
self.assertEqual(11, len(self.cat.get_resources(workspaces=[topp.name, sf.name])))
self.assertEqual("states", self.cat.get_resources(names="states", stores=states.name)[0].name)
self.assertEqual("states", self.cat.get_resources(names="states", workspaces=topp.name)[0].name)
self.assertEqual("states", self.cat.get_resources(names="states")[0].name)
self.assertEqual("states", self.cat.get_resource("states").name)
self.assertIsNone(self.cat.get_resource("blah+1blah-2"))
states = self.cat.get_resources(names="states")[0]
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", stores=sfdem.name)[0].name)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", workspaces=sf.name)[0].name)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem")[0].name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer":
continue
# looking for same name
ro = self.cat.get_resources(names=rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
count += 1
def testLayers(self):
if self.gs_version >= "2.13":
expected = set([
'sf:roads',
'sf:sfdem',
'nurc:mosaic',
'tiger:giant_polygon',
'sf:bugsites',
'topp:states',
'sf:streams',
'tiger:poly_landmarks',
'tiger:poi',
'topp:tasmania_water_bodies',
'tiger:tiger_roads',
'topp:tasmania_roads',
'nurc:Pk50095',
'topp:tasmania_cities',
'nurc:Img_Sample',
'sf:restricted',
'nurc:Arc_Sample',
'sf:archsites',
'topp:tasmania_state_boundaries'
])
else:
expected = set([
"Arc_Sample",
"Pk50095",
"Img_Sample",
"mosaic",
"sfdem",
"bugsites",
"restricted",
"streams",
"archsites",
"roads",
"tasmania_roads",
"tasmania_water_bodies",
"tasmania_state_boundaries",
"tasmania_cities",
"states",
"poly_landmarks",
"tiger_roads",
"poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups(names=["tasmania", "tiger-ny", "spearfish"]))
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroups(names="tasmania")[0]
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
if self.gs_version >= "2.13":
self.assertEqual(tas.layers, [
'topp:tasmania_state_boundaries',
'topp:tasmania_water_bodies',
'topp:tasmania_roads',
'topp:tasmania_cities'
], tas.layers)
else:
self.assertEqual(tas.layers, [
'tasmania_state_boundaries',
'tasmania_water_bodies',
'tasmania_roads',
'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assert_(self.cat.get_workspaces("topp")[0] is not None)
tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp")
self.cat.save(tas2)
self.assertEqual(1, len(self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")))
tas2 = self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")[0]
self.assert_("tasmania_reloaded", tas2.name)
self.assert_(isinstance(tas2, LayerGroup))
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
if self.gs_version >= "2.13":
self.assertEqual(tas2.layers, [
'topp:tasmania_state_boundaries',
'topp:tasmania_water_bodies',
'topp:tasmania_roads',
'topp:tasmania_cities'
], tas2.layers)
else:
self.assertEqual(tas2.layers, [
'tasmania_state_boundaries',
'tasmania_water_bodies',
'tasmania_roads',
'tasmania_cities'
], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_styles("population")[0].name)
self.assertEqual("popshade.sld", self.cat.get_styles("population")[0].filename)
self.assertEqual("population", self.cat.get_styles("population")[0].sld_name)
self.assertEqual("population", self.cat.get_style("population").sld_name)
self.assertIsNone(self.cat.get_style("blah+#5blah-"))
self.assertEqual(0, len(self.cat.get_styles('non-existing-style')))
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_styles("best style ever")
self.cat.get_workspaces("best workspace ever")
self.assertEqual(0, len(self.cat.get_stores(workspaces="best workspace ever", names="best store ever")))
self.cat.get_layer("best layer ever")
self.cat.get_layergroups("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
from geoserver.catalog import Catalog
cat = Catalog("http://localhost:8080/geoserver/rest")
print cat.get_layers()
# cat.create_workspace("the best workspace", "http://example.com/")
#
# d = cat.create_datastore("test datastore", "the best workspace")
# cat.save(d)
#
# for s in cat.get_workspaces():
# print s.name
#
# for s in cat.get_stores():
# print s.workspace.name, s.name
def handle(self, *args, **options):
rest_url = ogc_server_settings.rest
rest_user = ogc_server_settings.credentials.username
rest_pass = ogc_server_settings.credentials.password
dry_run = options['dry-run']
gs = Catalog(rest_url, rest_user, rest_pass)
layers = gs.get_layers()
if options['layer_list'] is not None:
layers = options['layer_list'].split(',')
unfixable = []
valid = []
def add_unfixable(layer_name, reason):
layer = {'name': layer_name,
'error': reason}
unfixable.append(layer)
def add_valid(layer_name, logged_attributes):
valid_layer = {'name': layer_name}
for key, value in logged_attributes.items():
valid_layer[key] = value
valid.append(valid_layer)
def check_valid_attribute(validation_object, object_to_check, attribute_to_check, default_value_to_check):
attribute = getattr(object_to_check, attribute_to_check)
if attribute is not None and attribute is not default_value_to_check:
validation_object[attribute_to_check] = attribute
return True
else:
return False
def fix_time_enabled(self, layer_object, validation_dict, dry_run):
resource = layer_object.resource
metadata = dict(resource.metadata)
metadata['time'].enabled = True
resource.metadata = metadata
if dry_run is False:
resource.catalog.save(resource)
check_layer = gs.get_layer(layer_object.name)
if check_valid_attribute(validation_dict, check_layer.resource.metadata.get('time'), 'enabled', False) is False:
self.stdout.write(
'Could not enable time for {0}'.format(layer_object.name))
else:
self.stdout.write(
'Time has been enabled for {0}'.format(layer_object.name))
def fix_time_presentation(self, layer_object, validation_dict, dry_run):
resource = layer_object.resource
metadata = dict(resource.metadata)
metadata['time'].presentation = 'LIST'
resource.metadata = metadata
if dry_run is False:
resource.catalog.save(resource)
check_layer = gs.get_layer(layer_object.name)
if check_valid_attribute(validation_dict, check_layer.resource.metadata.get('time'), 'presentation', 'LIST') is False:
self.stdout.write(
'Could not set presentation for {0}'.format(layer_object.name))
else:
self.stdout.write(
'Presentation has been set to list for {0}'.format(layer_object.name))
def fix_time_attribute(self, layer_object, validation_dict, dry_run):
# find date fields
resource = layer_object.resource
metadata = dict(resource.metadata)
fields = requests.get(layer_object.resource.href.replace(
'.xml', '.json'), auth=(rest_user, rest_pass))
the_fields = fields.json()['featureType']['attributes']
dates = [field.get('name') for field in the_fields['attribute'] if 'Time' in field.get(
'binding') or 'Date' in field.get('binding')]
if len(dates) == 0:
add_unfixable(layer_object.name,
'Layer does not contain a time attribute')
return
elif len(dates) == 1:
metadata['time'].attribute = dates[0]
else:
title = 'More than one date field found for {0} please select which to use for time attribute'.format(
layer_object.name)
option, index = pick(dates, title)
metadata['time'].attribute = dates[index]
resource.metadata = metadata
if dry_run is False:
resource.catalog.save(resource)
check_layer = gs.get_layer(layer_object.name)
if check_valid_attribute(validation_dict, check_layer.resource.metadata.get('time'), 'attribute', None) is False:
self.stdout.write(
'Could not set attribute for {0}'.format(layer_object.name))
else:
self.stdout.write('Attribute set to {0} for {1}'.format(
metadata['time'].attribute, layer_object.name))
for lyr in layers:
lyr_obj = lyr
if type(lyr) is str:
lyr_obj = gs.get_layer(lyr)
if lyr_obj is None:
add_unfixable(lyr, 'Layer does not exist')
self.stdout.write('{} does not exist in geoserver'.format(lyr))
continue
layer_validation = dict()
if lyr_obj.resource.metadata:
dimension_info = lyr_obj.resource.metadata.get('time')
if dimension_info is not None:
if check_valid_attribute(layer_validation, dimension_info, 'enabled', False) is False:
fix_time_enabled(
self, lyr_obj, layer_validation, dry_run)
if check_valid_attribute(layer_validation, dimension_info, 'presentation', 'LIST') is False:
fix_time_presentation(
self, lyr_obj, layer_validation, dry_run)
if check_valid_attribute(layer_validation, dimension_info, 'attribute', None) is False:
fix_time_attribute(
self, lyr_obj, layer_validation, dry_run)
else:
add_unfixable(
lyr, 'Layer was not uploaded with time configured.')
add_valid(lyr, layer_validation)
self.print_invalid_list(unfixable)
class GeoServer(Server):
"""
Represents a server running GeoServer [1], an application that provides access to layers.
This class provides a concrete implementation of the more generic Server component (which is intentionally generic).
Currently the Server class does not dictate an interface for accessing resources but this class aims to present
GeoServer specific components (such as workspaces) as generic components (such as namespaces).
GeoServer instances typically represent individual instances (i.e. hosts are servers) rather than a wider and more
abstract platform offered by a service provider.
Information on layers and other resources are fetched using a combination of the GeoServer specific administrative
API [2] accessed through geoserver-restconfig [3] and OGC services accessed through OWSLib [4] (and currently
limited to WMS and WFS).
[1] https://geoserver.readthedocs.io/en/latest/
[2] https://geoserver.readthedocs.io/en/latest/rest/index.html
[3] https://pypi.org/project/geoserver-restconfig
[4] https://pypi.org/project/OWSLib/
"""
def __init__(
self,
server_id: str,
label: str,
hostname: str,
port: str,
api_path: str,
wms_path: str,
wfs_path: str,
username: str,
password: str,
):
"""
:param server_id: unique identifier, typically a ULID (Universally Unique Lexicographically Sortable Identifier)
:param label: a human readable, well-known, identifier for the server - typically based on the hostname
:param hostname: servers fully qualified hostname
:param port: port on which GeoServer is running (usually '80' or '8080')
:param api_path: URL path, relative to the root of the server, to the GeoServer API (usually '/geoserver/rest')
:param wms_path: URL path, relative to the root of the server, to the GeoServer WMS endpoint (usually
'/geoserver/ows?service=wms&version=1.3.0&request=GetCapabilities')
:param wfs_path: URL path, relative to the root of the server, to the GeoServer WFS endpoint (usually
'/geoserver/ows?service=wfs&version=2.0.0&request=GetCapabilities')
:param username: username for account to use for GeoServer API
:param password: password for account to use for GeoServer API
"""
endpoint = build_base_data_source_endpoint(data_source={
"hostname": hostname,
"port": port
})
self.client = Catalogue(service_url=f"{endpoint}{api_path}",
username=username,
password=password)
self.wms = WebMapService(url=f"{endpoint}{wms_path}",
version="1.3.0",
username=username,
password=password)
self.wfs = WebFeatureService(url=f"{endpoint}{wfs_path}",
version="2.0.0",
username=username,
password=password)
super().__init__(
server_id=server_id,
label=label,
hostname=hostname,
server_type=ServerType.GEOSERVER.value,
version=self._get_geoserver_version(),
)
def get_namespaces(self) -> List[str]:
"""
Gets all GeoServer workspace names as Namespace labels
:return: list of Namespace labels
"""
workspaces = []
for workspace in self.client.get_workspaces():
workspaces.append(workspace.name)
return workspaces
def get_namespace(self, namespace_reference: str) -> Dict[str, str]:
"""
Gets a specific workspace as a Namespace
Note: GeoServer workspaces do not support the concept of a title, a static substitute value is therefore used
Note: GeoServer workspaces do support the concept of a namespace, but it is not yet implemented [#28]
:param namespace_reference: Namespace (workspace) label (name)
:return: dictionary of Namespace information that can be made into a Namespace object
"""
workspace = self.client.get_workspace(name=namespace_reference)
if workspace is None:
raise KeyError(
f"Namespace [{namespace_reference}] not found in server [{self.label}]"
)
return {"label": workspace.name, "title": "-", "namespace": "-"}
def get_repositories(self) -> List[Tuple[str, str]]:
"""
Gets all GeoServer store names as Repository labels
:return: list of Repository:Namespace label tuples
"""
stores = []
# Passing workspaces here is a workaround for a bug in the get stores method where workspaces aren't specified.
# The method says all workspaces should be checked but the logic to do this is in the wrong place so none are.
for store in self.client.get_stores(
workspaces=self.client.get_workspaces()):
stores.append((store.name, store.workspace.name))
return stores
def get_repository(self, repository_reference: str,
namespace_reference: str) -> Dict[str, str]:
"""
Gets a specific store as a Repository
If a Namespace (workspace) label is specified the Repository must exist within that Namespace.
GeoServer store types are sometimes unsuitable or non-standard and so need to be mapped to a conventional value.
in the RepositoryType enum using the GeoServerRepositoryType enum.
Note: GeoServer stores do not support the concept of a title, a static substitute value is therefore used
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param repository_reference: Repository (store) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of repository information that can be made into a Repository object
"""
_store = self.client.get_store(name=repository_reference,
workspace=namespace_reference)
if _store is None:
raise KeyError(
f"Repository [{repository_reference}] not found in server [{self.label}]"
)
store = {
"label":
_store.name,
"title":
"-",
"repository_type":
RepositoryType[GeoServerRepositoryType(str(
_store.type).lower()).name].value,
"namespace_label":
_store.workspace.name,
}
if hasattr(_store, "description") and _store.description is not None:
store["title"] = _store.description
if (store["repository_type"] == RepositoryType.POSTGIS.value
or store["repository_type"] == RepositoryType.ORACLE.value):
store["hostname"] = _store.connection_parameters["host"]
store["database"] = _store.connection_parameters["database"]
store["schema"] = _store.connection_parameters["schema"]
return store
def get_styles(self) -> List[Tuple[str, Optional[str]]]:
"""
Gets all GeoServer style names as Style labels
Python's None value will be used to represent the Namespace of global styles (i.e that don't have a Namespace
(workspace)).
:return: list of Style:Namespace label tuples
"""
styles = []
for _style in self.client.get_styles():
styles.append((_style.name, _style.workspace))
return styles
def get_style(self,
style_reference: str,
namespace_reference: str = None) -> Dict[str, str]:
"""
Gets a specific style as a Style
If a Namespace (workspace) label is specified the Style must exist within that Namespace.
Note: GeoServer styles do support the concept of a title, but it is not exposed through the admin API so a
static substitute value is therefore used
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param style_reference: Style (style) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of style information that can be made into a Style object
"""
_style = self.client.get_style(name=style_reference,
workspace=namespace_reference)
_type = str(_style.style_format).lower()
if _type == "sld10":
_type = "sld"
style = {
"label": _style.name,
"title": "-",
"style_type": _type,
}
if hasattr(_style, "workspace") and _style.workspace is not None:
style["namespace_label"] = _style.workspace
return style
def get_layers(self) -> List[str]:
"""
Gets all GeoServer layer names as Layer labels
:return: list of Layer labels
"""
layers = []
for _layer in self.client.get_layers():
layers.append(_layer.name)
return layers
def get_layer(
self, layer_reference: str
) -> Dict[str, Union[Optional[str], List[str], List[Tuple[
str, Optional[str]]]]]:
"""
Gets a specific layer as a Layer
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param layer_reference: Layer (layer) label (name)
:return: dictionary of layer information that can be made into a Layer object
"""
_layer = self.client.get_layer(name=layer_reference)
layer = {
"label":
_layer.resource.name,
"title":
_layer.resource.title,
"layer_type":
str(_layer.type).lower(),
"geometry_type":
None,
"services": [],
"table_view":
None,
"namespace_label":
_layer.resource.workspace.name,
"repository_label":
_layer.resource.store.name,
"style_labels":
[(_layer.default_style.name, _layer.default_style.workspace)],
}
if layer_reference in list(
self.wms.contents
) or f"{_layer.resource.workspace.name}:{layer_reference}" in list(
self.wms.contents):
layer["services"].append(LayerService.WMS.value)
if layer_reference in list(
self.wfs.contents
) or f"{_layer.resource.workspace.name}:{layer_reference}" in list(
self.wfs.contents):
layer["services"].append(LayerService.WFS.value)
# WFS lookups don't seem to mind if the layer is namespaced or not
_properties = self.wfs.get_schema(layer_reference)
if "geometry" in _properties and isinstance(
_properties["geometry"], str):
try:
layer["geometry_type"] = LayerGeometry[
GeoServerLayerGeometry(str(
_properties["geometry"])).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['geometry']}] for layer {layer_reference} not mapped to "
f"LayerGeometry enum.")
elif "properties" in _properties:
for geometry_column_name in GeoServerGeometryColumnNames:
if geometry_column_name.value in _properties[
"properties"].keys():
try:
layer["geometry_type"] = LayerGeometry[
GeoPropertyGeoServerLayerGeom(
str(_properties["properties"][
geometry_column_name.value])
).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['properties'][geometry_column_name.value]}] for layer "
f"{layer_reference} in column '{geometry_column_name.value}' not mapped to "
f"LayerGeometry enum.")
if (str(_layer.resource.store.type).lower()
== RepositoryType.POSTGIS.value
or str(_layer.resource.store.type).lower()
== RepositoryType.ORACLE.value):
layer["table_view"] = _layer.resource.native_name
return layer
def get_layer_groups(self) -> List[Tuple[str, Optional[str]]]:
"""
Gets all GeoServer layer group names as LayerGroup labels
Python's None value will be used to represent the Namespace of global layer groups (i.e that don't have a
Namespace (workspace)).
:return: list of LayerGroup:Namespace label tuples
"""
layer_groups = []
for _layer_group in self.client.get_layergroups(
workspaces=self.client.get_workspaces()):
layer_groups.append((_layer_group.name, _layer_group.workspace))
return layer_groups
def get_layer_group(
self, layer_group_reference: str, namespace_reference: str
) -> Dict[str, Union[Optional[str], List[str], List[Tuple[
str, Optional[str]]]]]:
"""
Gets a specific layer group as a LayerGroup
If a Namespace (workspace) label is specified the LayerGroup must exist within that Namespace.
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param layer_group_reference: LayerGroup (layer group) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of layer group information that can be made into a LayerGroup object
"""
_layer_group = self.client.get_layergroup(
name=layer_group_reference, workspace=namespace_reference)
layer_group = {
"label": _layer_group.name,
"title": _layer_group.title,
"services": [],
"namespace_label": _layer_group.workspace,
"layer_labels": [],
"style_labels": [],
}
for layer_label in _layer_group.layers:
layer_label = layer_label.split(":")
if len(layer_label) == 2:
layer_group["layer_labels"].append(
(layer_label[1], layer_label[0]))
elif len(layer_label) == 1:
layer_group["layer_labels"].append((layer_label[0], None))
if f"{namespace_reference}:{layer_group_reference}" in list(
self.wms.contents):
layer_group["services"].append(LayerService.WMS.value)
if f"{namespace_reference}:{layer_group_reference}" in list(
self.wfs.contents):
layer_group["services"].append(LayerService.WFS.value)
_properties = self.wfs.get_schema(
f"{namespace_reference}:{layer_group_reference}")
try:
layer_group["geometry_type"] = LayerGeometry[
GeoServerLayerGeometry(str(
_properties["geometry"])).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['geometry']}] not mapped to LayerGeometry enum."
)
for style_label in _layer_group.styles:
if style_label is not None:
style_label = style_label.split(":")
if len(style_label) == 2 and (
style_label[1],
style_label[0]) not in layer_group["style_labels"]:
layer_group["style_labels"].append(
(style_label[1], style_label[0]))
if len(style_label) == 1 and (
style_label[0],
None) not in layer_group["style_labels"]:
layer_group["style_labels"].append((style_label[0], None))
return layer_group
def _get_geoserver_version(self) -> str:
"""
Gets the GeoServer version
:return: GeoServer version string
"""
return self.client.get_version()
cat.reload()
passedTime += 1
# ensure AdvancedProjectionSetting is turned off (if not -> layers wont display layers correctly)???
r_set_wms_options = session.put(geoserver_url + '/services/wms/settings',
data='<wms><metadata><entry key="advancedProjectionHandling">false</entry></metadata></wms>',
headers=headers_xml)
# Delete old workspace and create new one
if cat.get_workspace(workspace):
if cat.get_store(projectName, workspace):
test=cat.get_store(projectName, workspace)
cat.delete(cat.get_store(projectName, workspace=workspace), purge="all", recurse=True)
cat.delete(cat.get_workspace(workspace), purge="all", recurse=True)
for layer in cat.get_layers():
if layer.name.startswith(workspace):
cat.delete(layer, recurse=True)
for style in cat.get_styles(workspaces=[workspace]):
cat.delete(style, recurse=True)
cat.create_workspace(workspace, geoserver_url + '/' + workspace)
# zip the ncFile
zfile = cfg['general']['workdir'] + '/outputFiles/data.zip'
logging.info('Writing Zipfile ' + zfile)
output = zipfile.ZipFile(zfile, 'w')
output.write(netcdfFile, projectName + '.nc', zipfile.ZIP_DEFLATED)
output.close()
# upload zip file (creating coveragestore and layers automatically)
logging.info('Uploading ' + zfile)
class ArchiveAndCatalogueI(geocloud.ArchiveAndCatalogue):
def __init__(self, com):
if not com:
raise RuntimeError("Not communicator")
self.com = com
q = com.stringToProxy('IceGrid/Query')
self.query = IceGrid.QueryPrx.checkedCast(q)
if not self.query:
raise RuntimeError("Invalid proxy")
self.broker = geocloud.BrokerPrx.checkedCast(
com.stringToProxy("broker"))
self.sem = Ice.threading.RLock(
) #Lock for managing the data structures
geoserver_path = "http://localhost:80/geoserver/rest"
self.catalog = Catalog(geoserver_path)
self.workspace = None
def setBroker(self, broker, current=None):
self.broker = broker
def log(self, log):
if self.broker:
self.broker.begin_appendLog("<ArchiveAndCatalogue> " + str(log))
else:
print log
def createScenario(self, scenario, current=None):
self.sem.acquire()
try:
if self.catalog == None:
self.log("Catalog")
raise geocloud.CreationScenarioException()
try:
self.workspace = self.catalog.create_workspace(
scenario, scenario)
self.workspace.enabled = True
self.log("Created Scenario %s" % (scenario))
except Exception:
print("workspace does not created")
self.log("Workspace does not created")
except FailedRequestError:
self.log("Request failed")
raise geocloud.CreationScenarioException()
finally:
self.sem.release()
def catalogue(self, path, store, scenario, current=None):
if self.workspace == None:
raise geocloud.CataloguingException()
else:
try:
if self.catalog == None:
raise geocloud.CataloguingException()
# store = path.split('/')[-1]
self.sem.acquire()
cv = self.catalog.create_coveragestore(store, path,
self.workspace)
self.log("%s Catalogued!" % (path))
except ConflictingDataError as e:
self.log(e)
except UploadError as e:
self.log(e)
except FailedRequestError as e:
None
except Exception as e:
self.log(e)
finally:
self.sem.release()
def deleteScenario(self, scenario, current=None):
self.sem.acquire()
try:
for layer in self.catalog.get_layers():
self.catalog.delete(layer)
for coverage in self.catalog.get_stores():
for resource in coverage.get_resources():
catalog.delete(resource)
catalog.delete(coverage)
for vk in catalog.get_workspaces():
catalog.delete(wk)
except Exception:
self.log("Exception while cleanning scenario")
# Check if remote datastore exists on local system
for dataStoreObj in dataStoresObj:
if dataStoreObj.name == remoteDSName:
localDSObj = dataStoreObj
print "Datastore " + localDSObj.name + " already exists"
if not localDSObj:
localDSObj = cat.create_datastore(remoteDSName, localWSName)
localDSObj.connection_parameters.update(host=remoteHost, port=remotePort, \
database=remoteDatabase, user=remoteUser, passwd=remotePassword, dbtype=remoteDBType)
cat.save(localDSObj)
print "Datastore " + localDSObj.name + " created"
# Check if remote layer already exists on local system
localLayersObj = cat.get_layers()
for localLayerObj in localLayersObj:
if localLayerObj.resource.name == remoteLayerName:
localLayerName = remoteLayerName
print "Layer " + remoteLayerName + " already published"
if not localLayerName:
# Publish remote layer
featureType = cat.publish_featuretype(remoteLayerName, localDSObj, WGS84, srs=WGS84)
print "Published layer " + remoteLayerName
# Update layers in GeoSHAPE
os.system("sudo geoshape-config updatelayers")
from geoserver.catalog import Catalog
cat = Catalog("http://localhost:8080/geoserver/rest")
print cat.get_layers()
# cat.create_workspace("the best workspace", "http://example.com/")
#
# d = cat.create_datastore("test datastore", "the best workspace")
# cat.save(d)
#
# for s in cat.get_workspaces():
# print s.name
#
# for s in cat.get_stores():
# print s.workspace.name, s.name
class wrap_geoserver:
""" Geoserver (gsconfig) wrapper """
def __init__(self,
geoserver_name,
username=username,
password=password,
easy=False):
if geoserver_name in list(REST.keys()):
self.path = REST[geoserver_name]
else:
self.path = geoserver_name
self.wms = self.path.replace("rest/", "wms")
self.name = geoserver_name
self.catalog = Catalog(self.path, username, password)
if not easy:
self.layers = []
self.layer_names = []
for layer in self.catalog.get_layers():
self.layers.append(layer)
self.layer_names.append(layer.name)
self.stores = [store for store in self.catalog.get_stores()]
self.store_names = [store.name for store in self.stores]
styles = []
self.workspaces = []
self.workspace_names = []
for workspace in self.catalog.get_workspaces():
styles = styles + self.catalog.get_styles(workspace)
self.workspace_names.append(workspace._name)
self.workspaces.append(workspace)
self.styles = styles + [
style for style in self.catalog.get_styles()
]
self.style_names = [style.name for style in self.styles]
def unpack(self, workspace_name, store_type="datastore"):
layers_and_styles = {}
features = []
workspace = self.get_workspace(workspace_name)
if store_type == "datastore":
store_url = workspace.datastore_url
elif store_type == "coveragestore":
store_url = workspace.coveragestore_url
else:
print("No correct store given")
for datastore in tqdm(get(store_url, "name")):
url = "{}workspaces/{}/datastores/{}".format(
self.path, workspace.name, datastore)
features = features + get(url, between_quotes=True)
for feature in features:
layer_name = os.path.basename(feature).split(".")[0]
self.get_layer(self.get_slug(workspace.name, layer_name))
layers_and_styles[layer_name] = self.layer.default_style
setattr(self, workspace_name + "_data", layers_and_styles)
return layers_and_styles
def get_layer(self, layer, easy=False):
self.layer = self.catalog.get_layer(layer)
if not easy:
self.resource = self.layer.resource
self.layer_name = self.layer.resource.name
self.sld_name = self.layer.default_style.name
self.sld_body = self.layer.default_style.sld_body
self.layer_latlon_bbox = self.layer.resource.latlon_bbox
self.layer_title = self.layer.resource.title
self.layer_abstract = self.layer.resource.abstract
def get_store(self, layer):
self.store = self.layer.resource._store
def get_resource(self):
self.resource = self.catalog.get_resource(self.layer.name, self.store)
def get_workspace(self, workspace_name):
self.workspace = self.catalog.get_workspace(workspace_name)
self.workspace_name = self.workspace._name
return self.workspace
def write_abstract(self, data, load_resource=True):
if load_resource:
self.get_resource()
self.resource.abstract = data
self.catalog.save(self.resource)
def write_title(self, title):
self.resource.title = title
self.catalog.save(self.resource)
def get_connection_parameters(self):
self.get_resource()
return self.resource.store.connection_parameters
def create_workspace(self, workspace_name):
workspace_exists = workspace_name in self.workspace_names
if not workspace_exists:
self.workspace = self.catalog.create_workspace(workspace_name)
else:
print("workspace already exists, using existing workspace")
self.workspace = self.catalog.get_workspace(workspace_name)
self.workspace_name = workspace_name
def create_postgis_datastore(self, store_name, workspace_name, pg_data):
try:
self.store = self.catalog.get_store(store_name,
self.workspace_name)
print("store within workspace exists, using existing store")
except Exception as e:
print(e)
ds = self.catalog.create_datastore(store_name, workspace_name)
ds.connection_parameters.update(
host=pg_data["host"],
port=pg_data["port"],
database=pg_data["database"],
user=pg_data["username"],
passwd=pg_data["password"],
dbtype="postgis",
schema="public",
)
self.save(ds)
self.store = self.catalog.get_store(store_name,
self.workspace_name)
self.store_name = store_name
def publish_layer(self,
layer_name,
workspace_name,
overwrite=False,
epsg="3857",
reload=False):
layer_exists = layer_name in self.layer_names
# if layer_name in self.workspace_layers[workspace_name]:
slug = self.get_slug(workspace_name, layer_name)
if overwrite and layer_exists:
print("Layer exists, deleting layer")
try:
self.layer = self.catalog.get_layer(slug)
self.delete(self.layer)
self.reload()
layer_exists = False
except Exception as e:
print(e)
print("Layer does not exist in workspace")
layer_exists = False
if not layer_exists:
feature_type = self.catalog.publish_featuretype(
layer_name,
self.store,
"EPSG:{}".format(str(epsg)),
srs="EPSG:{}".format(str(epsg)),
)
self.save(feature_type)
self.feature_type = feature_type
else:
print("layer already exists, using existing layer")
if reload:
self.get_layer(slug)
self.layer_name = layer_name
def publish_layergroup(self,
name,
layers,
styles=(),
bounds=None,
workspace=None):
layer_group = self.catalog.create_layergroup(name, layers, styles,
bounds, workspace)
self.save(layer_group)
def save(self, save_object):
return self.catalog.save(save_object)
def close(self):
self.catalog = None
def delete(self, delete_object):
self.catalog.delete(delete_object)
def reload(self):
self.catalog.reload()
def upload_shapefile(self, layer_name, shapefile_path):
path = shapefile_path.split(".shp")[0]
shapefile = shapefile_and_friends(path)
ft = self.catalog.create_featurestore(layer_name, shapefile,
self.workspace)
self.save(ft)
def upload_sld(self, sld_name, workspace_name, sld, overwrite=True):
style_exists = sld_name in self.style_names
if overwrite and style_exists:
print("Overwriting style")
style = self.catalog.get_style(sld_name, workspace_name)
self.delete(style)
self.reload()
style_exists = False
if not style_exists:
try:
self.catalog.create_style(sld_name, sld, False, workspace_name,
"sld11")
except Exception as e:
print(e)
style = self.catalog.get_style(sld_name, workspace_name)
self.delete(style)
self.reload()
self.catalog.create_style(sld_name, sld, False, workspace_name,
"sld10")
self.style_name = sld_name
else:
if style_exists:
print("Style already exists, using current style")
self.style_name = sld_name
def set_sld_for_layer(self,
workspace_name=None,
style_name=None,
use_custom=False):
if not use_custom:
workspace_name = self.workspace_name
style_name = self.style_name
self.style_slug = self.get_slug(workspace_name, style_name)
else:
if workspace_name is None:
self.style_slug = style_name
else:
self.style_slug = self.get_slug(workspace_name, style_name)
self.style = self.catalog.get_style(self.style_slug)
print("Setting {} for {}".format(self.style.name, self.layer.name))
self.layer.default_style = self.style
self.save(self.layer)
def get_slug(self, workspace, name):
return "{}:{}".format(workspace, name)
def get_slug_data(self, slug):
workspace_name = slug.split(":")[0]
layer_name = slug.split(":")[1]
return workspace_name, layer_name
def get_sld(self, layer_slug=None):
if layer_slug is None:
self.style = self.catalog.get_style(self.layer_slug)
else:
self.style = self.catalog.get_style(layer_slug)
self.sld_body = self.style.sld_body
return self.sld_body
def get_layer_workspace(self, layer_name):
return self.catalog.get_layer(layer_name).resource.workspace.name
def handle(self, *args, **options):
rest_url = ogc_server_settings.rest
rest_user = ogc_server_settings.credentials.username
rest_pass = ogc_server_settings.credentials.password
dry_run = options['dry-run']
gs = Catalog(rest_url, rest_user, rest_pass)
layers = gs.get_layers()
if options['layer_list'] is not None:
layers = options['layer_list'].split(',')
unfixable = []
valid = []
def add_unfixable(layer_name, reason):
layer = {'name': layer_name, 'error': reason}
unfixable.append(layer)
def add_valid(layer_name, logged_attributes):
valid_layer = {'name': layer_name}
for key, value in logged_attributes.items():
valid_layer[key] = value
valid.append(valid_layer)
def check_valid_attribute(validation_object, object_to_check,
attribute_to_check, default_value_to_check):
attribute = getattr(object_to_check, attribute_to_check)
if attribute is not None and attribute is not default_value_to_check:
validation_object[attribute_to_check] = attribute
return True
else:
return False
def fix_time_enabled(self, layer_object, validation_dict, dry_run):
resource = layer_object.resource
metadata = dict(resource.metadata)
metadata['time'].enabled = True
resource.metadata = metadata
if dry_run is False:
resource.catalog.save(resource)
check_layer = gs.get_layer(layer_object.name)
if check_valid_attribute(validation_dict,
check_layer.resource.metadata.get('time'),
'enabled', False) is False:
self.stdout.write('Could not enable time for {0}'.format(
layer_object.name))
else:
self.stdout.write('Time has been enabled for {0}'.format(
layer_object.name))
def fix_time_presentation(self, layer_object, validation_dict,
dry_run):
resource = layer_object.resource
metadata = dict(resource.metadata)
metadata['time'].presentation = 'LIST'
resource.metadata = metadata
if dry_run is False:
resource.catalog.save(resource)
check_layer = gs.get_layer(layer_object.name)
if check_valid_attribute(validation_dict,
check_layer.resource.metadata.get('time'),
'presentation', 'LIST') is False:
self.stdout.write('Could not set presentation for {0}'.format(
layer_object.name))
else:
self.stdout.write(
'Presentation has been set to list for {0}'.format(
layer_object.name))
def fix_time_attribute(self, layer_object, validation_dict, dry_run):
# find date fields
resource = layer_object.resource
metadata = dict(resource.metadata)
fields = requests.get(layer_object.resource.href.replace(
'.xml', '.json'),
auth=(rest_user, rest_pass))
the_fields = fields.json()['featureType']['attributes']
dates = [
field.get('name') for field in the_fields['attribute']
if 'Time' in field.get('binding')
or 'Date' in field.get('binding')
]
if len(dates) == 0:
add_unfixable(layer_object.name,
'Layer does not contain a time attribute')
return
elif len(dates) == 1:
metadata['time'].attribute = dates[0]
else:
title = 'More than one date field found for {0} please select which to use for time attribute'.format(
layer_object.name)
option, index = pick(dates, title)
metadata['time'].attribute = dates[index]
resource.metadata = metadata
if dry_run is False:
resource.catalog.save(resource)
check_layer = gs.get_layer(layer_object.name)
if check_valid_attribute(validation_dict,
check_layer.resource.metadata.get('time'),
'attribute', None) is False:
self.stdout.write('Could not set attribute for {0}'.format(
layer_object.name))
else:
self.stdout.write('Attribute set to {0} for {1}'.format(
metadata['time'].attribute, layer_object.name))
for lyr in layers:
lyr_obj = lyr
if type(lyr) is str:
lyr_obj = gs.get_layer(lyr)
if lyr_obj is None:
add_unfixable(lyr, 'Layer does not exist')
self.stdout.write('{} does not exist in geoserver'.format(lyr))
continue
layer_validation = dict()
if lyr_obj.resource.metadata:
dimension_info = lyr_obj.resource.metadata.get('time')
if dimension_info is not None:
if check_valid_attribute(layer_validation, dimension_info,
'enabled', False) is False:
fix_time_enabled(self, lyr_obj, layer_validation,
dry_run)
if check_valid_attribute(layer_validation, dimension_info,
'presentation', 'LIST') is False:
fix_time_presentation(self, lyr_obj, layer_validation,
dry_run)
if check_valid_attribute(layer_validation, dimension_info,
'attribute', None) is False:
fix_time_attribute(self, lyr_obj, layer_validation,
dry_run)
else:
add_unfixable(
lyr, 'Layer was not uploaded with time configured.')
add_valid(lyr, layer_validation)
self.print_invalid_list(unfixable)
def getLayer():
cat = Catalog("http://localhost:8081/geoserver/rest")
all_layers = cat.get_layers()
for layer in all_layers:
print(layer.name)
def publishGeoserver(appdef, progress):
viewCrs = appdef["Settings"]["App view CRS"]
usesGeoServer = False
for applayer in appdef["Layers"]:
if applayer.method != METHOD_FILE:
if applayer.layer.type() == applayer.layer.VectorLayer and applayer.layer.providerType().lower() != "wfs":
usesGeoServer = True
if not usesGeoServer:
return
progress.setText("Publishing to GeoServer")
progress.setProgress(0)
geoserverUrl = appdef["Deploy"]["GeoServer url"] + "/rest"
geoserverPassword = appdef["Deploy"]["GeoServer password"]
geoserverUsername = appdef["Deploy"]["GeoServer username"]
workspaceName = appdef["Deploy"]["GeoServer workspace"]
dsName = "ds_" + workspaceName
host = appdef["Deploy"]["PostGIS host"]
port = appdef["Deploy"]["PostGIS port"]
postgisUsername = appdef["Deploy"]["PostGIS username"]
postgisPassword = appdef["Deploy"]["PostGIS password"]
database = appdef["Deploy"]["PostGIS database"]
schema = appdef["Deploy"]["PostGIS schema"]
catalog = Catalog(geoserverUrl, geoserverUsername, geoserverPassword)
workspace = catalog.get_workspace(workspaceName)
if workspace is None:
workspace = catalog.create_workspace(workspaceName, workspaceName)
try:
store = catalog.get_store(dsName, workspace)
resources = store.get_resources()
for resource in resources:
layers = catalog.get_layers(resource)
for layer in layers:
catalog.delete(layer)
catalog.delete(resource)
catalog.delete(store)
except Exception:
pass
try:
store = catalog.get_store(dsName, workspace)
except FailedRequestError:
store = None
for i, applayer in enumerate(appdef["Layers"]):
layer = applayer.layer
if applayer.method != METHOD_FILE and applayer.method != METHOD_DIRECT:
name = safeName(layer.name())
sld, icons = getGsCompatibleSld(layer)
if sld is not None:
catalog.create_style(name, sld, True)
uploadIcons(icons, geoserverUsername, geoserverPassword, catalog.gs_base_url)
if layer.type() == layer.VectorLayer:
if applayer.method == METHOD_WFS_POSTGIS or applayer.method == METHOD_WMS_POSTGIS:
if store is None:
store = catalog.create_datastore(dsName, workspace)
store.connection_parameters.update(
host=host, port=str(port), database=database, user=postgisUsername, schema=schema,
passwd=postgisPassword, dbtype="postgis")
catalog.save(store)
catalog.publish_featuretype(name, store, layer.crs().authid())
else:
path = getDataFromLayer(layer, viewCrs)
catalog.create_featurestore(name,
path,
workspace=workspace,
overwrite=True)
gslayer = catalog.get_layer(name)
r = gslayer.resource
r.dirty['srs'] = viewCrs
catalog.save(r)
elif layer.type() == layer.RasterLayer:
path = getDataFromLayer(layer, viewCrs)
catalog.create_coveragestore(name,
path,
workspace=workspace,
overwrite=True)
if sld is not None:
publishing = catalog.get_layer(name)
publishing.default_style = catalog.get_style(name)
catalog.save(publishing)
progress.setProgress(int((i+1)*100.0/len(appdef["Layers"])))
def processAlgorithm(self, parameters, context, model_feedback):
"""
Process the algorithm
:param parameters: parameters of the process
:param context: context of the process
:param model_feedback: feedback instance for the process
:return:
"""
# Use a multi-step feedback, so that individual child algorithm progress reports are adjusted for the
# overall progress through the model
self.xml_path = parameters["XML_PATH"]
if not self.xml_path.lower().endswith(".xml"):
feedback = QgsProcessingMultiStepFeedback(0, model_feedback)
feedback.reportError(
"XML Workspace Definition is not an XML file!", True)
return {}
db_name = parameters["DB_NAME"]
db_params = self.get_db_params(db_name)
store_name = parameters["GS_STORE"]
wrk_name = parameters["GS_WORKSPACE"]
dataset_list = []
datasets = self.getDatasets()
for dataset in datasets:
type = dataset.getElementsByTagName(
TAG_DE_TYPE)[0].childNodes[0].data
if type == "esriDTFeatureClass":
ds_name = dataset.getElementsByTagName(
TAG_DE_NAME)[0].childNodes[0].data
dataset_list.append({
"name": ds_name.lower(),
"srs": "EPSG:4326"
})
feedback = QgsProcessingMultiStepFeedback(2 + len(dataset_list),
model_feedback)
feedback.pushInfo("Get GeoServer Catalog: " +
parameters["GS_REST_URL"])
gs_catalogue = Catalog(parameters["GS_REST_URL"],
parameters["GS_ADMIN"],
parameters["GS_PASSWORD"])
# workspace
if wrk_name == "" or wrk_name is None:
wrk_name = db_name.lower() + "_ws"
wrk_uri = "http://" + wrk_name
feedback.pushInfo("GeoServer Workspace: " + wrk_name + " (" + wrk_uri +
")")
workspace = gs_catalogue.get_workspace(wrk_name)
if workspace is None:
workspace = gs_catalogue.create_workspace(wrk_name, wrk_uri)
feedback.setCurrentStep(1)
# store
if store_name == "" or store_name is None:
store_name = db_name.lower() + "_ds"
feedback.pushInfo("GeoServer Data Store: " + store_name)
store = gs_catalogue.get_store(store_name, workspace)
if store is None:
store = gs_catalogue.create_datastore(store_name, workspace)
store.connection_parameters.update(**db_params)
gs_catalogue.save(store)
feedback.setCurrentStep(2)
step = 2
published_count = 0
layer_prefix = "v_"
for ds_cur in dataset_list:
layer_name = layer_prefix + ds_cur["name"]
feedback.pushInfo("GeoServer Publish: " + layer_name + " (" +
ds_cur["srs"] + ")")
try:
layer = gs_catalogue.get_layer(layer_name)
if layer is not None:
gs_catalogue.delete(layer)
gs_catalogue.save()
gs_catalogue.publish_featuretype(layer_name, store,
ds_cur["srs"])
published_count += 1
except Exception as e:
feedback.reportError("Error: " + str(e), False)
step += 1
feedback.setCurrentStep(step)
gs_catalogue.reload()
layers = gs_catalogue.get_layers(store)
feedback.pushInfo("-" * 80)
feedback.pushInfo("Published layers: " + str(published_count))
for layer in layers:
feedback.pushInfo(layer.name + " is published!")
feedback.pushInfo("-" * 80)
results = {}
outputs = {}
return results
class Client:
def __init__(self, geoserver, username, password):
self.restserver = urlparse.urljoin(geoserver, 'rest/')
self.wmsserver = urlparse.urljoin(geoserver, 'wms')
self.username = username
self.password = password
self.catalog = Catalog(self.restserver, self.username, self.password)
self.tempDir = tempfile.mkdtemp()
self.resource = None
self.layer = None
self.layerName = None
logging.basicConfig(
format="%(asctime)-15s %(name)-10s %(levelname)-7s : %(message)s",
level=logging.WARN)
self.logger = logging.getLogger("gsclient")
self.logger.setLevel(logging.DEBUG)
# setup logging for the gsclient
logging.getLogger('pyclowder').setLevel(logging.DEBUG)
logging.getLogger('__main__').setLevel(logging.DEBUG)
## this method assume that there is 1 store per layer
def getResourceByStoreName(self, storename, workspace):
if self.resource != None:
self.logger.debug("resource instance found; no need to fetch")
return self.resource
self.logger.debug("catalog.get_store called")
store = self.catalog.get_store(storename, workspace)
self.logger.debug("catalog.get_resources called based on store")
resources = self.catalog.get_resources(store=store)
self.logger.debug("fetched resources from server")
if resources == None:
return None
else:
self.resource = resources[0]
return self.resource
def getLayers(self):
layers = self.catalog.get_layers()
return layers
def getLayerByStoreName(self, storename):
self.logger.debug("getLayerbystore name started")
layers = self.catalog.get_layers()
for layer in layers:
if layer.resource.store.name == storename:
self.logger.debug("found the layer by store name")
return layer
return None
def getLayerByResource(self, resource):
if self.layer != None:
self.logger.debug("layer instance found; no need to fetch")
return self.layer
self.logger.debug("get Layer by Resource started...")
layers = self.catalog.get_layers(resource)
self.logger.debug("fetched layers from the server")
if layers == None:
return None
else:
self.layer = layers[0]
return self.layer
def mintMetadataWithoutGeoserver(self, workspace, filename, extent):
self.logger.debug("Creating wms metadata ... ")
metadata = {}
wmsLayerName = workspace + ':' + filename
metadata['WMS Layer Name'] = wmsLayerName
metadata['WMS Service URL'] = self.wmsserver
metadata[
'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng'
self.logger.debug('[DONE]')
return metadata
def mintMetadata(self, workspace, storename, extent):
self.logger.debug("Creating wms metadata ... ")
metadata = {}
layername = None
if self.layerName == None:
if self.layer == None:
self.logger.debug("getResourceByStoreName..")
resource = self.getResourceByStoreName(storename, workspace)
self.logger.debug("getLayerByResource ...")
layer = self.getLayerByResource(resource)
#layername = layer.name
self.logger.debug("done getting layer name")
if layer == None:
self.logger.debug('No layer found [DONE]')
return metadata
else:
layername = layer.name
else:
layername = self.layer.name
self.layerName = self.layer.name
else:
layername = self.layerName
# generate metadata
wmsLayerName = workspace + ':' + layername
metadata['WMS Layer Name'] = wmsLayerName
metadata['WMS Service URL'] = self.wmsserver
metadata[
'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng'
self.logger.debug('[DONE]')
return metadata
def uploadShapefile(self, geoserver_url, workspace, storename, filename,
projection, secret_key, proxy_on):
self.logger.debug("Uploading shapefile" + filename + "...")
if (proxy_on.lower() == 'true'):
self.logger.debug("proxy set to on ....")
# TODO activate proxy_on method if the proxy in clowder works
return self.geoserver_manipulation_proxy_off(
geoserver_url, workspace, storename, filename, projection)
# return self.geoserver_manipulation_proxy_on(geoserver_url, workspace, storename, filename, projection, secret_key)
else:
return self.geoserver_manipulation_proxy_off(
geoserver_url, workspace, storename, filename, projection)
def geoserver_manipulation_proxy_off(self, geoserver_url, workspace,
storename, filename, projection):
self.logger.debug("start geoserver manipulation....")
# create workspace if not present
is_workspace = False
self.logger.debug("checking workspace %s" % workspace)
response_worksp = requests.get(self.restserver + 'workspaces/' +
workspace,
auth=(self.username, self.password))
if response_worksp.status_code != 200:
new_worksp = "<workspace><name>" + workspace + "</name></workspace>"
response_worksp = requests.post(
self.restserver + 'workspaces',
headers={"Content-type": "text/xml"},
auth=(self.username, self.password),
data=new_worksp)
if response_worksp.status_code == 201:
is_workspace = True
else:
is_workspace = True
if is_workspace:
url = self.restserver + "workspaces/" + workspace + "/datastores/" + storename + "/file.shp"
response = None
self.logger.debug("put file to geosever %s" % url)
with open(filename, 'rb') as f:
response = requests.put(
url,
headers={'content-type': 'application/zip'},
auth=(self.username, self.password),
data=f)
self.logger.debug(str(response.status_code) + " " + response.text)
if response.status_code != 201:
self.logger.debug("[DONE]")
return False
self.set_projection(storename, workspace, projection)
return True
else:
return False
def geoserver_manipulation_proxy_on(self, geoserver_url, workspace,
storename, filename, projection,
secret_key):
# create workspace if not present
is_workspace = False
# this is a direct method, if the proxy works, this should go through proxy
last_charactor = geoserver_url[-1]
if last_charactor == '/':
geoserver_rest = geoserver_url + 'rest'
else:
geoserver_rest = geoserver_url + '/rest'
response_worksp = requests.get(geoserver_rest + '/workspaces/' +
workspace + '?key=' + secret_key,
auth=(self.username, self.password))
if response_worksp.status_code != 200:
new_worksp = "<workspace><name>" + workspace + "</name></workspace>"
response_worksp = requests.post(
geoserver_rest + '/workspaces' + '?key=' + secret_key,
headers={"Content-type": "text/xml"},
auth=(self.username, self.password),
data=new_worksp)
if response_worksp.status_code == 201:
is_workspace = True
else:
is_workspace = True
if is_workspace:
url = geoserver_rest + "/workspaces/" + workspace + "/datastores/" + storename + "/file.shp"
response = None
with open(filename, 'rb') as f:
response = requests.put(
url + '?key=' + secret_key,
headers={'content-type': 'application/zip'},
data=f)
self.logger.debug(str(response.status_code) + " " + response.text)
if response.status_code != 201:
self.logger.debug("[DONE]")
return False
self.set_projection(storename, workspace, projection)
return True
else:
return False
def set_projection(self, storename, workspace, projection):
resource = self.getResourceByStoreName(storename, workspace)
if resource.projection == None:
self.logger.debug('Setting projection' + projection)
resource.projection = projection
self.catalog.save(resource)
self.logger.debug("[DONE]")
self.layerName = storename
def createThumbnail(self, workspace, storename, extent, width, height):
self.logger.debug('Creating Thumbnail ...')
layername = None
if self.layerName == None:
if self.layer == None:
self.logger.debug("getResourceByStoreName..")
resource = self.getResourceByStoreName(storename, workspace)
self.logger.debug("getLayerByResource ...")
layer = self.getLayerByResource(resource)
self.logger.debug("done getting layer name")
if layer == None:
self.logger.debug('No layer found [DONE]')
return metadata
else:
layername = layer.name
else:
self.logger.debug("layer instance found: no need to fetch")
layername = self.layer.name
self.layerName = self.layer.name
else:
self.logger.debug("layerName instance found: no need to fetch")
layername = self.layerName
#wmsLayerName = workspace+":"+layer.name
wmsLayerName = workspace + ":" + layername
url = self.wmsserver + "?request=GetMap&layers=" + wmsLayerName + "&bbox=" + extent + "&width=" + width + "&height=" + height + "&srs=EPSG:3857&format=image%2Fpng"
r = requests.get(url, stream=True)
path = os.path.join(self.tempDir, 'tmp.png')
if r.status_code == 200:
tmp = r.headers['content-disposition']
tmplist = tmp.split(';')
for t in tmplist:
if t.strip().find('filename=') != -1:
path = os.path.join(self.tempDir, t.strip().split('=')[1])
with open(path, 'wb') as f:
for chunk in r.iter_content():
f.write(chunk)
self.logger.debug('[DONE]')
return path
else:
self.logger.debug('can not create thumbnail [DONE]')
return ''
def __del__(self):
# delete the temp dir
if self.tempDir != None:
try:
import shutil
self.logger.debug("Deleting temp dir " + self.tempDir)
shutil.rmtree(self.tempDir)
self.logger.debug("Deleted Temp file")
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
def __init__(self, gs_axx, dico_gs, tipo, txt=''):
"""Use OGR functions to extract basic informations about geoserver.
gs_axx = tuple like {url of a geoserver, user, password)
dico_gs = dictionary to store
tipo = format
text = dictionary of text in the selected language
"""
# connection
cat = Catalog(gs_axx[0],
gs_axx[1],
gs_axx[2],
disable_ssl_certificate_validation=gs_axx[3])
# print(dir(cat))
# -- WORKSPACES -------------------------------------------------------
workspaces = cat.get_workspaces()
for wk in workspaces:
# print(wk.name, wk.enabled, wk.resource_type, wk.wmsstore_url)
dico_gs[wk.name] = wk.href, {}
# print(dir(wk))
# -- STORES -----------------------------------------------------------
# stores = cat.get_stores()
# for st in stores:
# # print(st.name, st.enabled, st.href, st.resource_type)
# if hasattr(st, 'url'):
# url = st.url
# elif hasattr(st, 'resource_url'):
# url = st.resource_url
# else:
# print(dir(st))
# dico_gs.get(st.workspace.name)[1][st.name] = {"ds_type": st.type,
# "ds_url": url}
# print(dir(st))
# print(st.url)
# print(st.workspace)
# print(dir(st.workspace))
# print(st.workspace.name)
# -- LAYERS -----------------------------------------------------------
# resources_target = cat.get_resources(workspace='ayants-droits')
layers = cat.get_layers()
logging.info("{} layers found".format(len(layers)))
dico_layers = OrderedDict()
for layer in layers:
# print(layer.resource_type)
lyr_title = layer.resource.title
lyr_name = layer.name
lyr_wkspace = layer.resource._workspace.name
if type(layer.resource) is Coverage:
lyr_type = "coverage"
elif type(layer.resource) is FeatureType:
lyr_type = "vector"
else:
lyr_type = type(layer.resource)
# a log handshake
logging.info("{} | {} | {} | {}".format(layers.index(layer),
lyr_type, lyr_name,
lyr_title))
# # METADATA LINKS #
# download link
md_link_dl = "{0}/geoserver/{1}/ows?request=GetFeature"\
"&service=WFS&typeName={1}%3A{2}&version=2.0.0"\
"&outputFormat=SHAPE-ZIP"\
.format(url_base,
lyr_wkspace,
lyr_name)
# mapfish links
md_link_mapfish_wms = "{0}/mapfishapp/?layername={1}"\
"&owstype=WMSLayer&owsurl={0}/"\
"geoserver/{2}/ows"\
.format(url_base,
lyr_name,
lyr_wkspace)
md_link_mapfish_wfs = "{0}/mapfishapp/?layername={1}"\
"&owstype=WFSLayer&owsurl={0}/"\
"geoserver/{2}/ows"\
.format(url_base,
lyr_name,
lyr_wkspace)
md_link_mapfish_wcs = "{0}/mapfishapp/?cache=PreferNetwork"\
"&crs=EPSG:2154&format=GeoTIFF"\
"&identifier={1}:{2}"\
"&url={0}/geoserver/ows?"\
.format(url_base,
lyr_wkspace,
lyr_name)
# OC links
md_link_oc_wms = "{0}/geoserver/{1}/wms?layers={1}:{2}"\
.format(url_base,
lyr_wkspace,
lyr_name)
md_link_oc_wfs = "{0}/geoserver/{1}/ows?typeName={1}:{2}"\
.format(url_base,
lyr_wkspace,
lyr_name)
md_link_oc_wcs = "{0}/geoserver/{1}/ows?typeName={1}:{2}"\
.format(url_base,
lyr_wkspace,
lyr_name)
# CSW Querier links
md_link_csw_wms = "{0}/geoserver/ows?service=wms&version=1.3.0"\
"&request=GetCapabilities".format(url_base)
md_link_csw_wfs = "{0}/geoserver/ows?service=wfs&version=2.0.0"\
"&request=GetCapabilities".format(url_base)
# # # SERVICE LINKS #
# GeoServer Edit links
gs_link_edit = "{}/geoserver/web/?wicket:bookmarkablePage="\
":org.geoserver.web.data.resource."\
"ResourceConfigurationPage"\
"&name={}"\
"&wsName={}".format(url_base,
lyr_wkspace,
lyr_name)
# Metadata links (service => metadata)
if is_uuid(dict_match_gs_md.get(lyr_name)):
# HTML metadata
md_uuid_pure = dict_match_gs_md.get(lyr_name)
srv_link_html = "{}/portail/geocatalogue?uuid={}"\
.format(url_base, md_uuid_pure)
# XML metadata
md_uuid_formatted = "{}-{}-{}-{}-{}".format(
md_uuid_pure[:8], md_uuid_pure[8:12], md_uuid_pure[12:16],
md_uuid_pure[16:20], md_uuid_pure[20:])
srv_link_xml = "http://services.api.isogeo.com/ows/s/"\
"{1}/{2}?"\
"service=CSW&version=2.0.2&request=GetRecordById"\
"&id=urn:isogeo:metadata:uuid:{0}&"\
"elementsetname=full&outputSchema="\
"http://www.isotc211.org/2005/gmd"\
.format(md_uuid_formatted,
csw_share_id,
csw_share_token)
# add to GeoServer layer
rzourc = cat.get_resource(lyr_name,
store=layer.resource._store.name)
rzourc.metadata_links = [
('text/html', 'ISO19115:2003', srv_link_html),
('text/xml', 'ISO19115:2003', srv_link_xml),
('text/html', 'TC211', srv_link_html),
('text/xml', 'TC211', srv_link_xml)
]
# rzourc.metadata_links.append(('text/html', 'other', 'hohoho'))
cat.save(rzourc)
else:
logging.info("Service without metadata: {} ({})".format(
lyr_name, dict_match_gs_md.get(lyr_name)))
pass
dico_layers[layer.name] = {
"title": lyr_title,
"workspace": lyr_wkspace,
"store_name": layer.resource._store.name,
"store_type": layer.resource._store.type,
"lyr_type": lyr_type,
"md_link_dl": md_link_dl,
"md_link_mapfish": md_link_mapfish_wms,
"md_link_mapfish_wms": md_link_mapfish_wms,
"md_link_mapfish_wfs": md_link_mapfish_wfs,
"md_link_mapfish_wcs": md_link_mapfish_wcs,
"md_link_oc_wms": md_link_oc_wms,
"md_link_oc_wfs": md_link_oc_wfs,
"md_link_oc_wcs": md_link_oc_wcs,
"md_link_csw_wms": md_link_csw_wms,
"md_link_csw_wfs": md_link_csw_wfs,
"gs_link_edit": gs_link_edit,
"srv_link_html": srv_link_html,
"srv_link_xml": srv_link_xml,
"md_id_matching": md_uuid_pure
}
# mem clean up
# del dico_layer
# print(dico_gs.get(layer.resource._workspace.name)[1][layer.resource._store.name])
# print(dir(layer.resource))
# print(dir(layer.resource.writers))
# print(dir(layer.resource.writers.get("metadataLinks").func_dict))
# print(layer.resource.metadata)
# print(layer.resource.metadata_links)
dico_gs["layers"] = dico_layers
class ArchiveAndCatalogueI(geocloud.ArchiveAndCatalogue):
def __init__(self,com):
if not com:
raise RuntimeError("Not communicator")
self.com=com
q = com.stringToProxy('IceGrid/Query')
self.query = IceGrid.QueryPrx.checkedCast(q)
if not self.query:
raise RuntimeError("Invalid proxy")
self.broker=geocloud.BrokerPrx.checkedCast(com.stringToProxy("broker"))
self.sem = Ice.threading.RLock() #Lock for managing the data structures
geoserver_path = "http://localhost:80/geoserver/rest"
self.catalog = Catalog(geoserver_path)
self.workspace=None
def setBroker(self,broker,current=None):
self.broker =broker
def log(self,log):
if self.broker:
self.broker.begin_appendLog("<ArchiveAndCatalogue> "+str(log))
else:
print log
def createScenario(self,scenario,current=None):
self.sem.acquire()
try:
if self.catalog ==None:
self.log("Catalog")
raise geocloud.CreationScenarioException()
try:
self.workspace = self.catalog.create_workspace(scenario,scenario)
self.workspace.enabled =True
self.log("Created Scenario %s"%(scenario))
except Exception:
print ("workspace does not created")
self.log("Workspace does not created")
except FailedRequestError:
self.log("Request failed")
raise geocloud.CreationScenarioException();
finally:
self.sem.release()
def catalogue(self,path,store,scenario,current=None):
if self.workspace ==None:
raise geocloud.CataloguingException()
else:
try:
if self.catalog==None:
raise geocloud.CataloguingException()
# store = path.split('/')[-1]
self.sem.acquire()
cv = self.catalog.create_coveragestore(store,path,self.workspace)
self.log("%s Catalogued!"%(path))
except ConflictingDataError as e:
self.log(e)
except UploadError as e:
self.log(e)
except FailedRequestError as e:
None
except Exception as e:
self.log(e)
finally:
self.sem.release()
def deleteScenario(self,scenario,current=None):
self.sem.acquire()
try:
for layer in self.catalog.get_layers():
self.catalog.delete(layer)
for coverage in self.catalog.get_stores():
for resource in coverage.get_resources():
catalog.delete(resource)
catalog.delete(coverage)
for vk in catalog.get_workspaces():
catalog.delete(wk)
except Exception:
self.log("Exception while cleanning scenario")
logging.captureWarnings(True)
logger.setLevel(logging.INFO) # all errors will be get
log_form = logging.Formatter("%(asctime)s || %(levelname)s "
"|| %(module)s || %(message)s")
logfile = RotatingFileHandler("LOG_infos_geoserver.log", "a", 10000000, 2)
logfile.setLevel(logging.INFO)
logfile.setFormatter(log_form)
logger.addHandler(logfile)
logger.info('=================================================')
# ############################################################################
# ########## MAIN ##################
# ##################################
# connection
cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver")
# READ --------------------------------------------------------------------
all_layers = cat.get_layers()
print("Total layers count: " + len(all_layers))
# WRITE -------------------------------------------------------------------
# publish a shapefiles
wks_esipe = cat.get_workspace("ESIPE_IG3")
input_shapefiles = shapefile_and_friends(
path.realpath(r"data_samples\idf_RGP_emploi_exh"))
feat = cat.create_featurestore("test_gsoncif_write",
workspace=wks_esipe,
data=input_shapefiles)
cat = Catalog("http://localhost:8080/geoserver/rest")
resources = cat.get_resources()
gliStores = cat.get_stores()
iWorkspaces = cat.get_workspaces()
for loWorkspace in iWorkspaces:
print loWorkspace.name
print
for loStore in gliStores:
print loStore.name
print
iLayers = cat.get_layers()
for ilLayer in iLayers:
ilNomeLayer = ilLayer.name
print ilNomeLayer
print
for ilLayer in iLayers:
ilNomeLayer = ilLayer.name
if ilNomeLayer == 'ports_eu':
ilLayerSelezionatoRisorsa = cat.get_resource(ilNomeLayer)
print ilLayerSelezionatoRisorsa.attributes
