def remove_service(request, service_id): """ Delete a service, and its constituent layers. """ service_obj = get_object_or_404(Service, pk=service_id) if not request.user.has_perm("maps.delete_service", obj=service_obj): return HttpResponse( loader.render_to_string( "401.html", RequestContext(request, {"error_message": _("You are not permitted to remove this service.")}), ), status=401, ) if request.method == "GET": return render_to_response("services/service_remove.html", RequestContext(request, {"service": service_obj})) elif request.method == "POST": # Retrieve this service's workspace from the GeoServer catalog. cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password) workspace = cat.get_workspace(service_obj.name) # Delete nested workspace structure from GeoServer for this service. if workspace: for store in cat.get_stores(workspace): for resource in cat.get_resources(store): for layer in cat.get_layers(resource): cat.delete(layer) cat.delete(resource) cat.delete(store) cat.delete(workspace) # Delete service from GeoNode. service_obj.delete() return HttpResponseRedirect(reverse("services"))
def geoserver_upload(self, input_file): print "upload", input_file path, file_ext = os.path.split(input_file) f = os.path.splitext(file_ext)[0] cat = Catalog(self.GEOSERVER_HOST, self.GEOSERVER_USER, self.GEOSERVER_PASSWORD) try: cat.create_coveragestore(name=f, data=input_file, workspace=cat.get_workspace(self.GEOSERVER_WORKSPACE), overwrite=True) except UploadError: print "map already exist" cat.delete(cat.get_layer(f)) cat.delete(cat.get_resources(f, self.GEOSERVER_WORKSPACE)[0]) cat.create_coveragestore(name=f, data=input_file, workspace=cat.get_workspace(self.GEOSERVER_WORKSPACE), overwrite=True)
def deleteTempLayersGeoserver(cf): cat = Catalog( "{}/rest".format(cf.get("GeoServer", "host")), username=cf.get("GeoServer", "user"), password=cf.get("GeoServer", "pass"), ) my_resources = cat.get_resources(stores=["temp"], workspaces=["ra2ce-dr"]) counter = 0 for l in my_resources: name = l.name layer = cat.get_layer(name) # check if layer exists if not layer == None: logging.info("deleted:", name) cat.delete(layer) counter += 1 return counter
def remove_service(request, service_id): """ Delete a service, and its constituent layers. """ service_obj = get_object_or_404(Service, pk=service_id) if not request.user.has_perm('maps.delete_service', obj=service_obj): return HttpResponse(loader.render_to_string('401.html', RequestContext(request, { 'error_message': _("You are not permitted to remove this service.") })), status=401) if request.method == 'GET': return render_to_response("services/service_remove.html", RequestContext(request, { "service": service_obj })) elif request.method == 'POST': # Retrieve this service's workspace from the GeoServer catalog. cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) workspace = cat.get_workspace(service_obj.name) # Delete nested workspace structure from GeoServer for this service. if workspace: for store in cat.get_stores(workspace): for resource in cat.get_resources(store): for layer in cat.get_layers(resource): cat.delete(layer) cat.delete(resource) cat.delete(store) cat.delete(workspace) # Delete service from GeoNode. service_obj.delete() return HttpResponseRedirect(reverse("services"))
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name)
def gs_slurp( ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None, skip_unadvertised=False, remove_deleted=False, ): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ # avoid circular import problem from geonode.layers.models import set_attributes if console is None: console = open(os.devnull, "w") if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." cat = Catalog(ogc_server_settings.internal_rest, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if remove_deleted: resources_for_delete_compare = resources[:] workspace_for_delete_compare = workspace # filter out layers for delete comparison with GeoNode layers by following criteria: # enabled = true, if --skip-unadvertised: advertised = true, but disregard the filter parameter in the case of deleting layers resources_for_delete_compare = [k for k in resources_for_delete_compare if k.enabled == "true"] if skip_unadvertised: resources_for_delete_compare = [ k for k in resources_for_delete_compare if k.advertised == "true" or k.advertised == None ] if filter: resources = [k for k in resources if filter in k.name] # filter out layers depending on enabled, advertised status: resources = [k for k in resources if k.enabled == "true"] if skip_unadvertised: resources = [k for k in resources if k.advertised == "true" or k.advertised == None] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = {"stats": {"failed": 0, "updated": 0, "created": 0, "deleted": 0}, "layers": [], "deleted_layers": []} start = datetime.datetime.now() for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create( name=name, defaults={ "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name.encode("utf-8"), resource.name.encode("utf-8")), "title": resource.title or "No title provided", "abstract": resource.abstract or "No abstract provided", "owner": owner, "uuid": str(uuid.uuid4()), }, ) layer.save() # recalculate the layer statistics set_attributes(layer, overwrite=True) except Exception, e: if ignore_errors: status = "failed" exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >>sys.stderr, msg raise Exception("Failed to process %s" % resource.name.encode("utf-8"), e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = "created" output["stats"]["created"] += 1 else: status = "updated" output["stats"]["updated"] += 1 msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number) info = {"name": name, "status": status} if status == "failed": output["stats"]["failed"] += 1 info["traceback"] = traceback info["exception_type"] = exception_type info["error"] = error output["layers"].append(info) if verbosity > 0: print >> console, msg
''' Created on Mar 21, 2014 @author: lanalfa ''' from geoserver.catalog import Catalog import pprint cat = Catalog("http://localhost:8080/geoserver/rest") resources = cat.get_resources() gliStores = cat.get_stores() iWorkspaces = cat.get_workspaces() for loWorkspace in iWorkspaces: print loWorkspace.name print for loStore in gliStores: print loStore.name print iLayers = cat.get_layers() for ilLayer in iLayers: ilNomeLayer = ilLayer.name print ilNomeLayer print
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testAbout(self): about_html = self.cat.about() self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") self.cat.get_store(workspace="best workspace ever", name="best store ever") self.assertRaises(FailedRequestError, lambda: self.cat.get_resource( workspace="best workspace ever", store="best store ever", name="best resource ever")) self.cat.get_layer("best layer ever") self.cat.get_layergroup("best layergroup ever") def testUnicodeUrl(self): """ Tests that the geoserver.support.url function support unicode strings. """ # Test the url function with unicode seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml'] u = url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml") # Test the url function with normal string seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'] u = url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) self.gs_version = self.cat.get_short_version() def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resources("bugsites", workspaces="sf")[0] old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resources("bugsites", workspaces="sf")[0] self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resources("bugsites", workspaces="sf")[0] self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resources("bugsites", workspaces="sf")[0] self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resources("bugsites", workspaces="sf")[0] self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') data_source_name = lyr.resource.native_name # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_stores("gsconfig_import_test")[0] # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326", native_name=data_source_name) # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_stores("gsconfig_import_test")[0] lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. try: if lyr: self.cat.delete(lyr) self.cat.delete(lyr.resource) if ds: self.cat.delete(ds) except: pass def testDataStoreModify(self): ds = self.cat.get_stores("sf")[0] self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_stores("sf")[0] self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_stores("gsconfig_import_test")[0] self.cat.add_data_to_store( ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) @drop_table('import2') def testVirtualTables(self): ds = self.cat.create_datastore("gsconfig_import_test2") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_stores("gsconfig_import_test2")[0] self.cat.add_data_to_store( ds, "import2", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) geom = JDBCVirtualTableGeometry('the_geom', 'MultiPolygon', '4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = "select * from import2 where 'STATE_NAME' = 'Illinois'" keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) ft = self.cat.publish_featuretype(ft_name, ds, epsg_code, jdbc_virtual_table=jdbc_vt) # DISABLED; this test works only in the very particular case # "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR # def testCoverageStoreCreate(self): # ds = self.cat.create_coveragestore2("coverage_gsconfig") # ds.data_url = "file:test/data/mytiff.tiff" # self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_stores("sfdem")[0] self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_stores("sfdem")[0] self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set( ['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample", workspaces="nurc")[0] self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspaces("wmstest")[0] wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_stores("wmsstore")[0] self.assertEqual(1, len(self.cat.get_stores(workspaces=wmstest.name))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) # Testing projection and projection policy changes changed_layer.projection = "EPSG:900913" changed_layer.projection_policy = "REPROJECT_TO_DECLARED" self.cat.save(changed_layer) self.cat._cache.clear() layer = self.cat.get_layer(changed_layer.name) self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy) self.assertEqual(layer.resource.projection, changed_layer.projection) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.items(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspaces("sf")[0] self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf.name) self.assert_( len(self.cat.get_resources("states_test", workspaces=sf.name)) > 0) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore( "states_test", shapefile_plus_sidecars, sf)) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = { 'title': 'Not the original attribution', 'width': '123', 'height': '321', 'href': 'http://www.georchestra.org', 'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg', 'type': 'image/jpeg' } # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = 'pophatch' lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # check count before tests (upload) count = len(self.cat.get_styles()) # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) self.cat._cache.clear() fred = self.cat.get_styles(names="fred")[0] self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) self.cat._cache.clear() fred = self.cat.get_styles("fred")[0] self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.cat._cache.clear() self.assert_(len(self.cat.get_styles("fred")) == 0) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) self.cat._cache.clear() fred = self.cat.get_styles("fred")[0] self.assertEqual("Fred", fred.sld_title) # compare count after upload self.assertEqual(count + 1, len(self.cat.get_styles())) # attempt creating a new style without "title" self.cat.create_style("notitle", open("test/notitle.sld").read()) self.cat._cache.clear() notitle = self.cat.get_styles("notitle")[0] self.assertEqual(None, notitle.sld_title) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") self.cat._cache.clear() jed = self.cat.get_styles(names="jed", workspaces="blarny") self.assert_(len(jed) == 0) jed = self.cat.get_styles(names="jed", workspaces="topp") self.assert_(len(jed) == 1) self.assertEqual("Fred", jed[0].sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") self.cat._cache.clear() jed = self.cat.get_styles(names="jed", workspaces="topp") self.assert_(len(jed) == 1) self.assertEqual("Ted", jed[0].sld_title) # delete style, verify non-existence self.cat.delete(jed[0], purge=True) self.assertEqual( 0, len(self.cat.get_styles(names="jed", workspaces="topp"))) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") self.cat._cache.clear() jed = self.cat.get_styles(names="jed", workspaces="topp") self.assertEqual("Fred", jed[0].sld_title) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") self.cat._cache.clear() styles = self.cat.get_styles(names="ned, zed", workspaces="topp") self.assertEqual(2, len(styles)) ned, zed = styles lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspaces("acme") self.assertEqual(0, len(ws)) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspaces("acme")[0] self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspaces("foo")[0] self.cat.delete(ws) ws = self.cat.get_workspaces("foo") self.assertEqual(0, len(ws)) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_stores('states_shapefile')[0] self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_stores('states_shapefile')[0] self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_stores('states_shapefile')[0] self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroups("tasmania")[0] if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads', 'topp:tasmania_cities' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state # tas.refresh() if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testImageMosaic(self): """ Test case for Issue #110 """ # testing the mosaic creation name = 'cea_mosaic' data = open('test/data/mosaic/cea.zip', 'rb') self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assert_(resource is not None) # delete granule from mosaic coverage = name store = self.cat.get_stores(name)[0] granules = self.cat.list_granules(coverage, store) self.assertEqual(1, len(granules['features'])) granule_id = name + '.1' self.cat.delete_granule(coverage, store, granule_id) granules = self.cat.list_granules(coverage, store) self.assertEqual(0, len(granules['features'])) ''' testing external Image mosaic creation ''' name = 'cea_mosaic_external' path = os.path.join(os.getcwd(), 'test/data/mosaic/external') self.cat.create_imagemosaic(name, path, workspace='topp') self.cat._cache.clear() resource = self.cat.get_layer("external").resource self.assert_(resource is not None) # add granule to mosaic granule_path = os.path.join( os.getcwd(), 'test/data/mosaic/granules/cea_20150102.tif') self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(2, len(granules['features'])) # add external granule to mosaic granule_path = os.path.join( os.getcwd(), 'test/data/mosaic/granules/cea_20150103.zip') self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(3, len(granules['features'])) # Delete store store = self.cat.get_stores(name)[0] self.cat.delete(store, purge=True, recurse=True) self.cat._cache.clear() def testTimeDimension(self): sf = self.cat.get_workspaces("sf")[0] files = shapefile_and_friends( os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer( 'boxes_with_end_date').resource # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute)
def edit_layers(workspace): cat = Catalog(url=str(os.environ['GEOSERVER_URL']), user=user, pw=password) r = cat.get_resources(store="geodata", workspace=workspace) print(r)
def add_data_in_django(workspace=None): """ Add the layer in the earthcare geoserver into the Django instance. This function assumes, that the data is also on the local machine in the DEST_FOLDER and uses the .xml file of every layer to get the needed data. :return: None """ print 'Start:', asctime() namespaces = {'gmd':"http://www.isotc211.org/2005/gmd", 'gco':"http://www.isotc211.org/2005/gco",'gml':"http://www.opengis.net/gml"} from layers.models import Layer, Contact from swos.models import WetlandLayer, Product, Wetland url = settings.GEOSERVER['earthcare']['URL'] geoserver_user = settings.GEOSERVER['earthcare']['USER'] geoserver_password = settings.GEOSERVER['earthcare']['PASSWORD'] cat = Catalog(url, username=geoserver_user, password=geoserver_password) loc_folder = settings.DEST_FOLDER url = settings.METADATA_URL gs_resources = cat.get_resources(workspace=workspace) missing_meta = [] for gs_resource in gs_resources: name = gs_resource.name if WetlandLayer.objects.filter(identifier=name).count(): print "The Layer {name} is already integrated into Django.".format(name=name) continue if 'SWOS' in name: print name, asctime() shape_data = {} shape_data['identifier'] = name shape_data['title'] = name.replace('SWOS_-_', '').replace('_', ' ') workspace_name = gs_resource.workspace.name name = gs_resource.name print workspace_name for (k,v) in {'LST':'LSTT','LULC':'_LULC_','LULCC_L':'LULCC','SWD':'SWD','SSM':'SSM','Water_Quality':'WQ','Watershed':'Watershed'}.items(): if v in name: product_name = k xml = os.path.join(settings.DEST_FOLDER,workspace_name,product_name,name)+'.xml' try: tree = ET.parse(xml) #namespaces = getNamespaces(xml) except IOError as e: print e missing_meta.append([name, xml, e]) continue shape_data['epsg'] = '4326' bounding_box = gs_resource.latlon_bbox[:4] if gs_resource.url_part_types == 'featuretypes': type_string = 'wfs' else: type_string = 'wcs' country, wetland_name = workspace_name.split('_') testsite_url = ''.join([url, workspace_name, '/']) wms_url = ''.join([testsite_url, 'wms?']) type_url = ''.join([testsite_url, type_string, '?']) shape_data['download_url'] = type_url png = 'png/{z}/{x}/{y}.png' tms_url = '{url}gwc/service/tms/1.0.0/{testsite}:{layername}@EPSG:900913@{png}' tms_url = tms_url.format(testsite=workspace_name, layername=name, png=png, url=url) shape_data['ogc_link'] = tms_url shape_data['ogc_type'] = 'TMS' legend_url = ''.join( [wms_url, 'REQUEST=GetLegendGraphic&VERSION=1.0.0&FORMAT=image/png&WIDTH=20&HEIGHT=20&LAYER=', workspace_name, ':', name, '&LEGEND_OPTIONS=forceLabels:on']) if product_name in ['LULC', 'LULCC_L']: sld = os.path.join(loc_folder, workspace_name, product_name,name)+'.sld' shape_data['legend_colors'] = get_rgb_json(sld,product_name) elif product_name =='SWD': sld=os.path.join(settings.SLD_FOLDER,'finals','SWD.sld') shape_data['legend_colors'] =get_rgb_json(sld,product_name) shape_data['legend_url'] = legend_url if product_name in ['Water_Quality', 'LULC']: wq_type = name.split('_')[2] else: wq_type = '' root =tree.getroot() abstract = root.find('.//gmd:abstract',namespaces) shape_data['abstract'] = abstract[0].text shape_data['west'] = bounding_box[0] shape_data['east'] = bounding_box[1] shape_data['south'] = bounding_box[2] shape_data['north'] = bounding_box[3] date_begin = root.find('.//gmd:EX_TemporalExtent//gml:beginPosition', namespaces).text date_end = root.find('.//gmd:EX_TemporalExtent//gml:endPosition', namespaces).text print date_begin, date_end shape_data['date_begin'] = datetime.datetime.strptime(date_begin, '%Y-%m-%d').date() shape_data['date_end'] = datetime.datetime.strptime(date_end, '%Y-%m-%d').date() if product_name == 'Watershed': date_string ='' elif product_name in ['LULC','SWD']: date_string = str(shape_data['date_begin'].year) elif product_name in ['LULCC_L','LST']: date_string = ' '.join([str(shape_data['date_begin'].year), 'to', str(shape_data['date_end'].year)]) elif product_name in ['Water_Quality', 'SSM']: date_string = shape_data['date_begin'].strftime('%Y %m') dataset_contact = root.find('.//gmd:contact',namespaces) dataset_email = dataset_contact.find('.//gmd:electronicMailAddress',namespaces) dataset_email = dataset_email[0].text try: dataset_contact = Contact.objects.get(email=dataset_email) shape_data['dataset_contact_new'] =dataset_contact except Contact.DoesNotExist as e: print 'The contact information for', gs_instance,'is missing.' missing_meta.append([name, dataset_email,e]) continue try: wetland = Wetland.objects.get(short_name=wetland_name) except Wetland.DoesNotExist as e: missing_meta.append([name, wetland_name, e]) continue shape_data['wetland'] = wetland product = Product.objects.get(short_name=product_name) print product print product_name print product.name print shape_data['date_begin'] product.wetlands.add(wetland) shape_data['product'] = product shape_data['title'] = ' '.join([product.name, wq_type, date_string, wetland.name, country]) print 'Title:', shape_data['title'] layer = WetlandLayer(**shape_data) layer.save() print shape_data print asctime() for thing in missing_meta: print thing print len(missing_meta) return
ooiDataStore = "ooi" ooiworkspace = "geonode" #update layer that_layer = cat.get_layer("1k time varying test") that_layer.enabled = False # at this point that_layer is still published in GeoServer cat.save(that_layer) # now it is disabled that_layer = cat.get_layer("1k time varying test") that_layer.enabled = True cat.save(that_layer) ooitest = cat.get_workspace("ooi_test") data = cat.get_resources(store="asd",workspace="geonode") print data layer = cat.get_layer("1k time varying test") print layer.href cat.add_data_to_store print layer #drop those things not of interest wmstest = cat.get_workspace("geonode") try: wmsstore = cat.get_store("ooi") cat.delete(wmsstore) except: print "opps"
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None, skip_unadvertised=False, remove_deleted=False): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ # avoid circular import problem from geonode.layers.models import set_attributes if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." cat = Catalog(ogc_server_settings.internal_rest, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) if workspace is None: resources = [] else: resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if remove_deleted: resources_for_delete_compare = resources[:] workspace_for_delete_compare = workspace # filter out layers for delete comparison with GeoNode layers by following criteria: # enabled = true, if --skip-unadvertised: advertised = true, but disregard the filter parameter in the case of deleting layers resources_for_delete_compare = [ k for k in resources_for_delete_compare if k.enabled == "true" ] if skip_unadvertised: resources_for_delete_compare = [ k for k in resources_for_delete_compare if k.advertised == "true" or k.advertised == True or k.advertised is None ] if filter: resources = [k for k in resources if filter in k.name] # filter out layers depending on enabled, advertised status: resources = [k for k in resources if k.enabled == "true"] if skip_unadvertised: resources = [ k for k in resources if k.advertised == "true" or k.advertised == True or k.advertised is None ] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = { 'stats': { 'failed': 0, 'updated': 0, 'created': 0, 'deleted': 0, }, 'layers': [], 'deleted_layers': [] } start = datetime.datetime.now() for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create( name=name, defaults={ "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() # recalculate the layer statistics set_attributes(layer, overwrite=True) except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception( 'Failed to process %s' % resource.name.encode('utf-8'), e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' output['stats']['created'] += 1 else: status = 'updated' output['stats']['updated'] += 1 msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number) info = {'name': name, 'status': status} if status == 'failed': output['stats']['failed'] += 1 info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output['layers'].append(info) if verbosity > 0: print >> console, msg
#!/usr/bin/env python """ gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . """ __author__ = "David Winslow" __copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") ws = cat.get_workspace("sf") resources = cat.get_resources(workspace=ws) if len(resources) != 0: assert all(r.projection == "EPSG:27613" for r in resources), ws.name
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2018 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") pg_stores = [s.name for s in cat.get_stores() if s.resource_type == 'dataStore' and s.connection_parameters.get("dbtype") == "postgis"] print(cat.get_resources(stores=pg_stores))
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None, skip_unadvertised=False): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ # avoid circular import problem from geonode.layers.models import set_attributes if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." cat = Catalog(ogc_server_settings.internal_rest, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if filter: resources = [k for k in resources if filter in k.name] # filter out layers depending on enabled, advertised status: resources = [k for k in resources if k.enabled == "true"] if skip_unadvertised: resources = [k for k in resources if k.advertised == "true" or k.advertised == None] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = { 'stats': { 'failed':0, 'updated':0, 'created':0, }, 'layers': [] } start = datetime.datetime.now() for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create(name=name, defaults = { "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() # recalculate the layer statistics set_attributes(layer, overwrite=True) except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception('Failed to process %s' % resource.name.encode('utf-8'), e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' output['stats']['created']+=1 else: status = 'updated' output['stats']['updated']+=1 msg = "[%s] Layer %s (%d/%d)" % (status, name, i+1, number) info = {'name': name, 'status': status} if status == 'failed': output['stats']['failed']+=1 info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output['layers'].append(info) if verbosity > 0: print >> console, msg
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2018 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") native_bbox = [ '589434.856', '4914006.338', '609527.21', '4928063.398', 'EPSG:26713' ] latlon_bbox = ['-103.877', '44.371', '-103.622', '44.5', 'EPSG:4326'] for rs in cat.get_resources(workspaces='sf'): rs.native_bbox = native_bbox rs.latlon_bbox = latlon_bbox cat.save(rs)
#!/usr/bin/env python from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") ws = cat.get_workspace("sf") resources = cat.get_resources(workspace=ws) if len(resources) != 0: assert all(r.projection == "EPSG:27613" for r in resources), ws.name
#!/usr/bin/env python from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") pg_stores = [s for s in cat.get_stores() if s.connection_parameters and \ s.connection_parameters.get("dbtype") == "postgis"] res = [] for s in pg_stores: res.extend(r.name for r in cat.get_resources(store=s)) print res
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2018 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") resources = cat.get_resources(workspaces='sf') if len(resources) > 0: assert all(r.projection == "EPSG:26713" for r in resources)
#!/usr/bin/env python from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") native_bbox = \ ['589434.856', '4914006.338', '609527.21', '4928063.398', 'EPSG:26713'] latlon_bbox = ['-103.877', '44.371', '-103.622', '44.5', 'EPSG:4326'] sf = cat.get_workspace('sf') for rs in cat.get_resources(workspace=sf): rs.native_bbox = native_bbox rs.latlon_bbox = latlon_bbox cat.save(rs)
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testAbout(self): about_html = self.cat.about() self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testResourcesUpdate(self): res_dest = self.cat.get_resources() count = 0 for rd in res_dest: # only wms layers if rd.resource_type != "wmsLayer": continue # looking for same name ro = self.cat.get_resource(rd.name) if ro is not None: rd.title = ro.title rd.abstract = ro.abstract rd.keywords = ro.keywords rd.projection = ro.projection rd.native_bbox = ro.native_bbox rd.latlon_bbox = ro.latlon_bbox rd.projection_policy = ro.projection_policy rd.enabled = ro.enabled rd.advertised = ro.advertised rd.metadata_links = ro.metadata_links or None self.cat.save(rd) self.cat.reload() # print "Updated layer: " + rd.name count += 1 # print "Total updated layers: " + str(count) def testLayers(self): expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) # Try to create a new Layer Group into the "topp" workspace self.assert_(self.cat.get_workspace("topp") is not None) tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp") self.cat.save(tas2) self.assert_(self.cat.get_layergroup("tasmania_reloaded") is None) self.assert_(self.cat.get_layergroup("tasmania_reloaded", "topp") is not None) tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp") self.assert_("tasmania_reloaded", tas2.name) self.assert_(isinstance(tas2, LayerGroup)) self.assertEqual(tas2.workspace, "topp", tas2.workspace) self.assertEqual(tas2.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas2.layers) self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles) def testStyles(self): self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assert_(self.cat.get_style('non-existing-style') is None) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") try: self.cat.get_store(workspace="best workspace ever", name="best store ever") self.fail('expected exception') except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message) try: self.cat.get_resource(workspace="best workspace ever", store="best store ever", name="best resource ever") except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message)
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") pg_stores = [s for s in cat.get_stores() if s.connection_parameters and \ s.connection_parameters.get("dbtype") == "postgis"] res = [] for s in pg_stores: res.extend(r.name for r in cat.get_resources(store=s)) print res
#! /usr/bin/env python from geoserver.catalog import Catalog cat = Catalog('http://host/geoserver/rest', 'user', 'password') def internal_name(s): return s.startswith("__") and s.endswith("__") def check(x): if hasattr(x, 'name'): print "NAME:", x.name, " TYPE:", type(x) for name in dir(x): if not internal_name(name) and getattr(x, name) is None: print x, name, "IS NONE" for ws in cat.get_workspaces(): check(ws) for st in cat.get_stores(): check(st) for rs in cat.get_resources(st): check(rs)
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." url = "%srest" % settings.OGC_SERVER['default']['LOCATION'] cat = Catalog(url, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if filter: resources = [k for k in resources if filter in k.name] # filter out layers explicitly disabled by geoserver resources = [k for k in resources if k.enabled == "true"] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = [] for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create( name=name, defaults={ "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception( 'Failed to process %s' % resource.name.encode('utf-8'), e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' else: status = 'updated' msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number) info = {'name': name, 'status': status} if status == 'failed': info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output.append(info) if verbosity > 0: print >> console, msg
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testAbout(self): about_html = self.cat.about() self.assertTrue( '<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assert_(self.cat.get_style('non-existing-style') is None) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") try: self.cat.get_store(workspace="best workspace ever", name="best store ever") self.fail('expected exception') except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message) try: self.cat.get_resource(workspace="best workspace ever", store="best store ever", name="best resource ever") except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message)
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testAbout(self): about_html = self.cat.about() self.assertTrue( '<html xmlns="http://www.w3.org/1999/xhtml"' in str(about_html)) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d+(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertTrue(pat.match('2.10.1')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) self.assertEqual("topp", self.cat.get_workspaces("topp")[-1].name) self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde']))) self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde'))) def testStores(self): self.assertEqual(0, len(self.cat.get_stores("nonexistentstore"))) topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(workspace=topp))) self.assertEqual(2, len(self.cat.get_stores(workspace=sf))) self.assertEqual(2, len(self.cat.get_stores(workspace='sf'))) self.assertEqual( 2, len(self.cat.get_stores(names='states_shapefile, sfdem'))) self.assertEqual( 2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem']))) self.assertEqual("sfdem", self.cat.get_stores("sfdem")[-1].name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", "topp").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testResourcesUpdate(self): res_dest = self.cat.get_resources() count = 0 for rd in res_dest: # only wms layers if rd.resource_type != "wmsLayer": continue # looking for same name ro = self.cat.get_resource(rd.name) if ro is not None: rd.title = ro.title rd.abstract = ro.abstract rd.keywords = ro.keywords rd.projection = ro.projection rd.native_bbox = ro.native_bbox rd.latlon_bbox = ro.latlon_bbox rd.projection_policy = ro.projection_policy rd.enabled = ro.enabled rd.advertised = ro.advertised rd.metadata_links = ro.metadata_links or None self.cat.save(rd) self.cat.reload() # print "Updated layer: " + rd.name count += 1 # print "Total updated layers: " + str(count) def testLayers(self): expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assertTrue(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assertEqual("states", states.name) self.assertIsInstance(states.resource, ResourceInfo) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assertTrue(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assertEqual("tasmania", tas.name) self.assertIsInstance(tas, LayerGroup) self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) # Try to create a new Layer Group into the "topp" workspace self.assertIsNotNone(self.cat.get_workspace("topp")) tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace="topp") self.cat.save(tas2) self.assertIsNone(self.cat.get_layergroup("tasmania_reloaded")) self.assertIsNotNone( self.cat.get_layergroup("tasmania_reloaded", "topp")) tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp") self.assertEqual("tasmania_reloaded", tas2.name) self.assertIsInstance(tas2, LayerGroup) self.assertEqual(tas2.workspace, "topp", tas2.workspace) self.assertEqual(tas2.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas2.layers) self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles) def testStyles(self): self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assertIsNone(self.cat.get_style('non-existing-style')) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") self.assertEqual( self.cat.get_store(workspace="best workspace ever", name="best store ever"), None) self.cat.get_layer("best layer ever") self.cat.get_layergroup("best layergroup ever") def testUnicodeUrl(self): """ Tests that the geoserver.support.url function support unicode strings. """ # Test the url function with unicode seg = [ 'workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml' ] u = url(base=self.cat.service_url, seg=seg) self.assertEqual( u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml") # Test the url function with normal string seg = [ 'workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml' ] u = url(base=self.cat.service_url, seg=seg) self.assertEqual( u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2018 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") resources = cat.get_resources(workspaces='sf') if len(resources) > 0: assert all(r.projection == "EPSG:26713" for r in resources)
class Client: def __init__(self, geoserver, username, password): self.restserver = urlparse.urljoin(geoserver, 'rest/') self.wmsserver = urlparse.urljoin(geoserver, 'wms') self.username = username self.password = password self.catalog = Catalog(self.restserver, self.username, self.password) self.tempDir = tempfile.mkdtemp() self.resource = None self.layer = None self.layerName = None logging.basicConfig( format="%(asctime)-15s %(name)-10s %(levelname)-7s : %(message)s", level=logging.WARN) self.logger = logging.getLogger("gsclient") self.logger.setLevel(logging.DEBUG) # setup logging for the gsclient logging.getLogger('pyclowder').setLevel(logging.DEBUG) logging.getLogger('__main__').setLevel(logging.DEBUG) ## this method assume that there is 1 store per layer def getResourceByStoreName(self, storename, workspace): if self.resource != None: self.logger.debug("resource instance found; no need to fetch") return self.resource self.logger.debug("catalog.get_store called") store = self.catalog.get_store(storename, workspace) self.logger.debug("catalog.get_resources called based on store") resources = self.catalog.get_resources(store=store) self.logger.debug("fetched resources from server") if resources == None: return None else: self.resource = resources[0] return self.resource def getLayers(self): layers = self.catalog.get_layers() return layers def getLayerByStoreName(self, storename): self.logger.debug("getLayerbystore name started") layers = self.catalog.get_layers() for layer in layers: if layer.resource.store.name == storename: self.logger.debug("found the layer by store name") return layer return None def getLayerByResource(self, resource): if self.layer != None: self.logger.debug("layer instance found; no need to fetch") return self.layer self.logger.debug("get Layer by Resource started...") layers = self.catalog.get_layers(resource) self.logger.debug("fetched layers from the server") if layers == None: return None else: self.layer = layers[0] return self.layer def mintMetadataWithoutGeoserver(self, workspace, filename, extent): self.logger.debug("Creating wms metadata ... ") metadata = {} wmsLayerName = workspace + ':' + filename metadata['WMS Layer Name'] = wmsLayerName metadata['WMS Service URL'] = self.wmsserver metadata[ 'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng' self.logger.debug('[DONE]') return metadata def mintMetadata(self, workspace, storename, extent): self.logger.debug("Creating wms metadata ... ") metadata = {} layername = None if self.layerName == None: if self.layer == None: self.logger.debug("getResourceByStoreName..") resource = self.getResourceByStoreName(storename, workspace) self.logger.debug("getLayerByResource ...") layer = self.getLayerByResource(resource) #layername = layer.name self.logger.debug("done getting layer name") if layer == None: self.logger.debug('No layer found [DONE]') return metadata else: layername = layer.name else: layername = self.layer.name self.layerName = self.layer.name else: layername = self.layerName # generate metadata wmsLayerName = workspace + ':' + layername metadata['WMS Layer Name'] = wmsLayerName metadata['WMS Service URL'] = self.wmsserver metadata[ 'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng' self.logger.debug('[DONE]') return metadata def uploadShapefile(self, geoserver_url, workspace, storename, filename, projection, secret_key, proxy_on): self.logger.debug("Uploading shapefile" + filename + "...") if (proxy_on.lower() == 'true'): self.logger.debug("proxy set to on ....") # TODO activate proxy_on method if the proxy in clowder works return self.geoserver_manipulation_proxy_off( geoserver_url, workspace, storename, filename, projection) # return self.geoserver_manipulation_proxy_on(geoserver_url, workspace, storename, filename, projection, secret_key) else: return self.geoserver_manipulation_proxy_off( geoserver_url, workspace, storename, filename, projection) def geoserver_manipulation_proxy_off(self, geoserver_url, workspace, storename, filename, projection): self.logger.debug("start geoserver manipulation....") # create workspace if not present is_workspace = False self.logger.debug("checking workspace %s" % workspace) response_worksp = requests.get(self.restserver + 'workspaces/' + workspace, auth=(self.username, self.password)) if response_worksp.status_code != 200: new_worksp = "<workspace><name>" + workspace + "</name></workspace>" response_worksp = requests.post( self.restserver + 'workspaces', headers={"Content-type": "text/xml"}, auth=(self.username, self.password), data=new_worksp) if response_worksp.status_code == 201: is_workspace = True else: is_workspace = True if is_workspace: url = self.restserver + "workspaces/" + workspace + "/datastores/" + storename + "/file.shp" response = None self.logger.debug("put file to geosever %s" % url) with open(filename, 'rb') as f: response = requests.put( url, headers={'content-type': 'application/zip'}, auth=(self.username, self.password), data=f) self.logger.debug(str(response.status_code) + " " + response.text) if response.status_code != 201: self.logger.debug("[DONE]") return False self.set_projection(storename, workspace, projection) return True else: return False def geoserver_manipulation_proxy_on(self, geoserver_url, workspace, storename, filename, projection, secret_key): # create workspace if not present is_workspace = False # this is a direct method, if the proxy works, this should go through proxy last_charactor = geoserver_url[-1] if last_charactor == '/': geoserver_rest = geoserver_url + 'rest' else: geoserver_rest = geoserver_url + '/rest' response_worksp = requests.get(geoserver_rest + '/workspaces/' + workspace + '?key=' + secret_key, auth=(self.username, self.password)) if response_worksp.status_code != 200: new_worksp = "<workspace><name>" + workspace + "</name></workspace>" response_worksp = requests.post( geoserver_rest + '/workspaces' + '?key=' + secret_key, headers={"Content-type": "text/xml"}, auth=(self.username, self.password), data=new_worksp) if response_worksp.status_code == 201: is_workspace = True else: is_workspace = True if is_workspace: url = geoserver_rest + "/workspaces/" + workspace + "/datastores/" + storename + "/file.shp" response = None with open(filename, 'rb') as f: response = requests.put( url + '?key=' + secret_key, headers={'content-type': 'application/zip'}, data=f) self.logger.debug(str(response.status_code) + " " + response.text) if response.status_code != 201: self.logger.debug("[DONE]") return False self.set_projection(storename, workspace, projection) return True else: return False def set_projection(self, storename, workspace, projection): resource = self.getResourceByStoreName(storename, workspace) if resource.projection == None: self.logger.debug('Setting projection' + projection) resource.projection = projection self.catalog.save(resource) self.logger.debug("[DONE]") self.layerName = storename def createThumbnail(self, workspace, storename, extent, width, height): self.logger.debug('Creating Thumbnail ...') layername = None if self.layerName == None: if self.layer == None: self.logger.debug("getResourceByStoreName..") resource = self.getResourceByStoreName(storename, workspace) self.logger.debug("getLayerByResource ...") layer = self.getLayerByResource(resource) self.logger.debug("done getting layer name") if layer == None: self.logger.debug('No layer found [DONE]') return metadata else: layername = layer.name else: self.logger.debug("layer instance found: no need to fetch") layername = self.layer.name self.layerName = self.layer.name else: self.logger.debug("layerName instance found: no need to fetch") layername = self.layerName #wmsLayerName = workspace+":"+layer.name wmsLayerName = workspace + ":" + layername url = self.wmsserver + "?request=GetMap&layers=" + wmsLayerName + "&bbox=" + extent + "&width=" + width + "&height=" + height + "&srs=EPSG:3857&format=image%2Fpng" r = requests.get(url, stream=True) path = os.path.join(self.tempDir, 'tmp.png') if r.status_code == 200: tmp = r.headers['content-disposition'] tmplist = tmp.split(';') for t in tmplist: if t.strip().find('filename=') != -1: path = os.path.join(self.tempDir, t.strip().split('=')[1]) with open(path, 'wb') as f: for chunk in r.iter_content(): f.write(chunk) self.logger.debug('[DONE]') return path else: self.logger.debug('can not create thumbnail [DONE]') return '' def __del__(self): # delete the temp dir if self.tempDir != None: try: import shutil self.logger.debug("Deleting temp dir " + self.tempDir) shutil.rmtree(self.tempDir) self.logger.debug("Deleted Temp file") except OSError as exc: if exc.errno != errno.ENOENT: raise
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) self.gs_version = self.cat.get_short_version() def testGSVersion(self): version = self.cat.get_version() pat = re.compile('\d\.\d+') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspaces(names="topp")[-1].name) self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde']))) self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde'))) self.assertEqual("topp", self.cat.get_workspace("topp").name) self.assertIsNone(self.cat.get_workspace("blahblah-")) def testStores(self): self.assertEqual(0, len(self.cat.get_stores(names="nonexistentstore"))) topp = self.cat.get_workspaces("topp")[0] sf = self.cat.get_workspaces("sf")[0] self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(workspaces=topp))) self.assertEqual(2, len(self.cat.get_stores(workspaces=sf))) self.assertEqual(2, len(self.cat.get_stores(workspaces='sf'))) self.assertEqual(2, len(self.cat.get_stores(names='states_shapefile, sfdem'))) self.assertEqual(2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem']))) self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0].name) self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile")[0].name) self.assertEqual("sfdem", self.cat.get_stores(names="sfdem", workspaces=sf.name)[0].name) self.assertEqual("sfdem", self.cat.get_stores(names="sfdem")[0].name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) self.assertIsNone(self.cat.get_store("blah+blah-")) def testResources(self): topp = self.cat.get_workspaces("topp")[0] sf = self.cat.get_workspaces("sf")[0] states = self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0] sfdem = self.cat.get_stores(names="sfdem", workspaces=sf.name)[0] self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(2, len(self.cat.get_resources(stores=[states.name, sfdem.name]))) self.assertEqual(11, len(self.cat.get_resources(workspaces=[topp.name, sf.name]))) self.assertEqual("states", self.cat.get_resources(names="states", stores=states.name)[0].name) self.assertEqual("states", self.cat.get_resources(names="states", workspaces=topp.name)[0].name) self.assertEqual("states", self.cat.get_resources(names="states")[0].name) self.assertEqual("states", self.cat.get_resource("states").name) self.assertIsNone(self.cat.get_resource("blah+1blah-2")) states = self.cat.get_resources(names="states")[0] fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", stores=sfdem.name)[0].name) self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", workspaces=sf.name)[0].name) self.assertEqual("sfdem", self.cat.get_resources(names="sfdem")[0].name) def testResourcesUpdate(self): res_dest = self.cat.get_resources() count = 0 for rd in res_dest: # only wms layers if rd.resource_type != "wmsLayer": continue # looking for same name ro = self.cat.get_resources(names=rd.name) if ro is not None: rd.title = ro.title rd.abstract = ro.abstract rd.keywords = ro.keywords rd.projection = ro.projection rd.native_bbox = ro.native_bbox rd.latlon_bbox = ro.latlon_bbox rd.projection_policy = ro.projection_policy rd.enabled = ro.enabled rd.advertised = ro.advertised rd.metadata_links = ro.metadata_links or None self.cat.save(rd) self.cat.reload() count += 1 def testLayers(self): if self.gs_version >= "2.13": expected = set([ 'sf:roads', 'sf:sfdem', 'nurc:mosaic', 'tiger:giant_polygon', 'sf:bugsites', 'topp:states', 'sf:streams', 'tiger:poly_landmarks', 'tiger:poi', 'topp:tasmania_water_bodies', 'tiger:tiger_roads', 'topp:tasmania_roads', 'nurc:Pk50095', 'topp:tasmania_cities', 'nurc:Img_Sample', 'sf:restricted', 'nurc:Arc_Sample', 'sf:archsites', 'topp:tasmania_state_boundaries' ]) else: expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups(names=["tasmania", "tiger-ny", "spearfish"])) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroups(names="tasmania")[0] self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads', 'topp:tasmania_cities' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) # Try to create a new Layer Group into the "topp" workspace self.assert_(self.cat.get_workspaces("topp")[0] is not None) tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp") self.cat.save(tas2) self.assertEqual(1, len(self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp"))) tas2 = self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")[0] self.assert_("tasmania_reloaded", tas2.name) self.assert_(isinstance(tas2, LayerGroup)) self.assertEqual(tas2.workspace, "topp", tas2.workspace) if self.gs_version >= "2.13": self.assertEqual(tas2.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads', 'topp:tasmania_cities' ], tas2.layers) else: self.assertEqual(tas2.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas2.layers) self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles) def testStyles(self): self.assertEqual("population", self.cat.get_styles("population")[0].name) self.assertEqual("popshade.sld", self.cat.get_styles("population")[0].filename) self.assertEqual("population", self.cat.get_styles("population")[0].sld_name) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assertIsNone(self.cat.get_style("blah+#5blah-")) self.assertEqual(0, len(self.cat.get_styles('non-existing-style'))) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_styles("best style ever") self.cat.get_workspaces("best workspace ever") self.assertEqual(0, len(self.cat.get_stores(workspaces="best workspace ever", names="best store ever"))) self.cat.get_layer("best layer ever") self.cat.get_layergroups("best layergroup ever") def testUnicodeUrl(self): """ Tests that the geoserver.support.url function support unicode strings. """ # Test the url function with unicode seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml'] u = build_url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml") # Test the url function with normal string seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'] u = build_url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) self.gs_version = self.cat.get_short_version() def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resources("bugsites")[0] old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resources("bugsites")[0] self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resources("bugsites")[0] self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resources("bugsites")[0] self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resources("bugsites")[0] self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') data_source_name = lyr.resource.native_name # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_stores("gsconfig_import_test")[0] # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326", native_name=data_source_name) # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_stores("gsconfig_import_test")[0] lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. try: if lyr: self.cat.delete(lyr) self.cat.delete(lyr.resource) if ds: self.cat.delete(ds) except: pass def testDataStoreModify(self): ds = self.cat.get_stores("sf")[0] self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_stores("sf")[0] self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_stores("gsconfig_import_test")[0] self.cat.add_data_to_store(ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) @drop_table('import2') def testVirtualTables(self): ds = self.cat.create_datastore("gsconfig_import_test2") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_stores("gsconfig_import_test2")[0] self.cat.add_data_to_store(ds, "import2", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) geom = JDBCVirtualTableGeometry('the_geom', 'MultiPolygon', '4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = "select * from import2 where 'STATE_NAME' = 'Illinois'" keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) ft = self.cat.publish_featuretype(ft_name, ds, epsg_code, jdbc_virtual_table=jdbc_vt) # DISABLED; this test works only in the very particular case # "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR # def testCoverageStoreCreate(self): # ds = self.cat.create_coveragestore2("coverage_gsconfig") # ds.data_url = "file:test/data/mytiff.tiff" # self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_stores("sfdem")[0] self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_stores("sfdem")[0] self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resources("Arc_Sample")[0] old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample")[0] self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample")[0] self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample")[0] self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample")[0] self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample")[0] self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resources("Arc_Sample")[0] self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspaces("wmstest")[0] wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_stores("wmsstore")[0] self.assertEqual(1, len(self.cat.get_stores(workspaces=wmstest.name))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) # Testing projection and projection policy changes changed_layer.projection = "EPSG:900913" changed_layer.projection_policy = "REPROJECT_TO_DECLARED" self.cat.save(changed_layer) self.cat._cache.clear() layer = self.cat.get_layer(changed_layer.name) self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy) self.assertEqual(layer.resource.projection, changed_layer.projection) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.items(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspaces("sf")[0] self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf.name) self.assert_(len(self.cat.get_resources("states_test", workspaces=sf.name)) > 0) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) ) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = { 'title': 'Not the original attribution', 'width': '123', 'height': '321', 'href': 'http://www.georchestra.org', 'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg', 'type': 'image/jpeg' } # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = 'pophatch' lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # check count before tests (upload) count = len(self.cat.get_styles()) # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) self.cat._cache.clear() fred = self.cat.get_styles(names="fred")[0] self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) self.cat._cache.clear() fred = self.cat.get_styles("fred")[0] self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.cat._cache.clear() self.assert_(len(self.cat.get_styles("fred")) == 0) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) self.cat._cache.clear() fred = self.cat.get_styles("fred")[0] self.assertEqual("Fred", fred.sld_title) # compare count after upload self.assertEqual(count + 1, len(self.cat.get_styles())) # attempt creating a new style without "title" self.cat.create_style("notitle", open("test/notitle.sld").read()) self.cat._cache.clear() notitle = self.cat.get_styles("notitle")[0] self.assertEqual(None, notitle.sld_title) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") self.cat._cache.clear() jed = self.cat.get_styles(names="jed", workspaces="blarny") self.assert_(len(jed) == 0) jed = self.cat.get_styles(names="jed", workspaces="topp") self.assert_(len(jed) == 1) self.assertEqual("Fred", jed[0].sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") self.cat._cache.clear() jed = self.cat.get_styles(names="jed", workspaces="topp") self.assert_(len(jed) == 1) self.assertEqual("Ted", jed[0].sld_title) # delete style, verify non-existence self.cat.delete(jed[0], purge=True) self.assertEqual(0, len(self.cat.get_styles(names="jed", workspaces="topp"))) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") self.cat._cache.clear() jed = self.cat.get_styles(names="jed", workspaces="topp") self.assertEqual("Fred", jed[0].sld_title) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") self.cat._cache.clear() styles = self.cat.get_styles(names="ned, zed", workspaces="topp") self.assertEqual(2, len(styles)) ned, zed = styles lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspaces("acme") self.assertEqual(0, len(ws)) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspaces("acme")[0] self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspaces("foo")[0] self.cat.delete(ws) ws = self.cat.get_workspaces("foo") self.assertEqual(0, len(ws)) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_stores('states_shapefile')[0] self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_stores('states_shapefile')[0] self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_stores('states_shapefile')[0] self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroups("tasmania")[0] if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads', 'topp:tasmania_cities' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testImageMosaic(self): """ Test case for Issue #110 """ # testing the mosaic creation name = 'cea_mosaic' data = open('test/data/mosaic/cea.zip', 'rb') self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assert_(resource is not None) # delete granule from mosaic coverage = name store = self.cat.get_stores(name)[0] granules = self.cat.list_granules(coverage, store) self.assertEqual(1, len(granules['features'])) granule_id = name + '.1' self.cat.delete_granule(coverage, store, granule_id) granules = self.cat.list_granules(coverage, store) self.assertEqual(0, len(granules['features'])) ''' testing external Image mosaic creation ''' name = 'cea_mosaic_external' path = os.path.join(os.getcwd(), 'test/data/mosaic/external') self.cat.create_imagemosaic(name, path, workspace = 'topp') self.cat._cache.clear() resource = self.cat.get_layer("external").resource self.assert_(resource is not None) # add granule to mosaic granule_path = os.path.join(os.getcwd(), 'test/data/mosaic/granules/cea_20150102.tif') self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(2, len(granules['features'])) # add external granule to mosaic granule_path = os.path.join(os.getcwd(), 'test/data/mosaic/granules/cea_20150103.zip') self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(3, len(granules['features'])) # Delete store store = self.cat.get_stores(name)[0] self.cat.delete(store, purge=True, recurse=True) self.cat._cache.clear() def testTimeDimension(self): sf = self.cat.get_workspaces("sf")[0] files = shapefile_and_friends(os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer('boxes_with_end_date').resource # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute)
password=password) errors = [] try: workspace = gscatalog.get_workspace(name=args.workspace) except SSLError as e: logger.error( "Unable to connect: SSL error (hint: use --disable-ssl-verification option)" ) sys.exit(1) if workspace is None: logger.error("Workspace \"%s\" not found" % args.workspace) sys.exit(1) else: resources = gscatalog.get_resources(workspace=workspace) for res in resources: try: # UUID from MDD is needed for operatesOn elements md_url, md = find_data_metadata(res, creds, args.disable_ssl_verification) layer = gscatalog.get_layer(name="%s:%s" % (res.workspace.name, res.name)) linked_md = guess_related_service_metadata( args.geoserver, args.geonetwork, args.workspace, args.service) if linked_md is None: # Creates a new service metadata for the workspace logger.info( "No service metadata found for %s, creating one", args.workspace)
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." url = "%srest" % settings.GEOSERVER_BASE_URL cat = Catalog(url, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if filter: resources = [k for k in resources if filter in k.name] # filter out layers explicitly disabled by geoserver resources = [k for k in resources if k.enabled == "true"] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = [] for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create(name=name, defaults = { "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name, resource.name), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception('Failed to process %s' % resource.name, e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' else: status = 'updated' msg = "[%s] Layer %s (%d/%d)" % (status, name, i+1, number) info = {'name': name, 'status': status} if status == 'failed': info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output.append(info) if verbosity > 0: print >> console, msg
from geoserver.catalog import Catalog cat = Catalog("http://172.18.77.15:8089/geoserver/rest", username="******", password="******") url = 'http://172.18.77.15:8089/geoserver/%s/wms?service=WMS&version=1.1.0&request=GetMap&layers=%s:%s&styles=&bbox=%s,%s,%s,%s&width=80&height=80&srs=%s&format=image/png' rr = cat.get_resources(workspace=cat.get_workspace('shenzhen')) for r in rr: print r.name,url%(r.workspace.name,r.workspace.name,r.name,r.native_bbox[0],r.native_bbox[2],\ r.native_bbox[1],r.native_bbox[3],r.native_bbox[4] if r.native_bbox[4] is not None \ and r.native_bbox[4].startswith('EPSG') else ('EPSG:4326' if float(r.native_bbox[0])<10000 else 'EPSG:2309' ) )