def delete_map(request): map_id = request.POST.get('map_id', False) cat = Catalog(map_url, 'admin', 'geoserver') if cat.get_layer('Map:' + map_id): cat.delete(cat.get_layer('Map:' + map_id)) cat.reload() for label_type in range(1, 8): if cat.get_layer('Mask:' + map_id + '_' + str(label_type)): cat.delete(cat.get_layer('Mask:' + map_id + '_' + str(label_type))) cat.reload() try: if cat.get_store(map_id): st = cat.get_store(map_id) cat.delete(st) cat.reload() except Exception: pass if Bmap.objects.filter(id=map_id): map_name = Bmap.objects.get(id=map_id).name dir_root = os.path.join(MAPBASEPATH, map_name) delete_files = (map_id + ".jpg", 'chaneltransform.tif', 'chaneltransform_rpc.txt', 'chaneltransformRPC.txt', 'chaneltransformRPC.tif.ovr', 'label.tif', 'label_rpc.txt', 'labelRPC.tif') for file in delete_files: if os.path.exists(file): os.remove(os.path.join(dir_root, file)) Bmap.objects.filter(id=map_id).delete() return HttpResponse("success")
def addLayersToGeoserver(self, options): cat = Catalog(self.geoserver_rest_url, options["geoserveradmin"], options["gpw"]) try: ds = cat.get_store(options["alias"]) except Exception as e: raise Exception("Erreur de récupération du workspace") layers = [] try: # connect to tables and create layers and correct urban styles for table in self.urb: try: style = self.urb[table] ft = cat.publish_featuretype(table, ds, "EPSG:31370", srs="EPSG:31370") gs_style = cat.get_style(style) cat.save(ft) res_name = ft.dirty["name"] res_title = options["alias"] + "_" + table cat.save(ft) layer_name = ds.workspace.name + ":" + res_name new_layer = cat.get_layer(layer_name) new_layer.default_style = gs_style cat.save(new_layer) layers.append({"res_name": res_name, "res_title": res_title}) except Exception as e: # a verifier une fois un possesion des styles print(str(e)) except Exception as e: print(str(e)) raise Exception("Erreur lors de la récupération des couches depuis Geoserver") return layers
def main(): parser = argparse.ArgumentParser( description="OWS12 Sentinel harvest granules script.") parser.add_argument('-c', '--catalog', nargs=3, help='Geoserver REST URL and authentication', default=('http://localhost:8080/geoserver/rest', 'admin', 'geoserver')) parser.add_argument( '-s', '--store', nargs=1, help='The store name to retrieve coverages with workspace name ' '(ows12:landsat)') parser.add_argument( '-i', '--insert', action="store_true", default=False, help='Harvest (insert) new granule on a given mosaic store') parser.add_argument('-d', '--delete', nargs=1, type=int, help='Delete granules older than given months value.') parser.add_argument('granules', help='Path to ingestion file.') args = parser.parse_args() sentinel = SentinelSat() # Before we proceed, check if we retrieved any files during previous processing step if not os.path.exists( os.path.join(args.granules, sentinel.processed_granules)): logger.info( 'Missing file from previous download job.\nPlease run this job first!' ) sys.exit(1) num_lines = sum(1 for l in open( os.path.join(args.granules, sentinel.processed_granules))) if num_lines == 0: logger.info('Skipping processing step. No files to process found!') sys.exit(0) catalog = Catalog(service_url=args.catalog[0], username=args.catalog[1], password=args.catalog[2]) store = catalog.get_store(args.store[0].split(':')[1], args.store[0].split(':')[0]) coverages = catalog.mosaic_coverages(store) # Harvest granules if args.insert: sentinel.insert_granules(args.granules, catalog, coverages, store) # Delete granules if args.delete: sentinel.delete_granules(catalog, coverages, store, args.delete)
def addLayersToGeoserver(self, options): cat = Catalog(self.geoserver_rest_url, options['geoserveradmin'], options['gpw']) try: ds = cat.get_store(options['alias']) except Exception as e: raise Exception('Erreur de récupération du workspace') layers = [] try: #connect to tables and create layers and correct urban styles for table in self.urb: try: style = self.urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') gs_style = cat.get_style(style) cat.save(ft) res_name = ft.dirty['name'] res_title = options['alias']+"_"+table cat.save(ft) layer_name = ds.workspace.name + ':' + res_name new_layer = cat.get_layer(layer_name) new_layer.default_style = gs_style cat.save(new_layer) layers.append({ 'res_name' : res_name, 'res_title' : res_title }) except Exception as e: # a verifier une fois un possesion des styles print(str(e)) except Exception as e: print(str(e)) raise Exception('Erreur lors de la récupération des couches depuis Geoserver') return layers
def main(options): #connect to geoserver cat = Catalog("http://localhost:8080/geoserver/rest", "admin", options.gpw) #create datrastore for URB schema ws = cat.create_workspace(options.alias,'imio.be') ds = cat.create_datastore(options.alias, ws) ds.connection_parameters.update( host=options.urbanUrl, port="5432", database=options.database, user="******", passwd=options.ropw, dbtype="postgis") cat.save(ds) ds = cat.get_store(options.alias) #config object urb = { "capa":"Parcelles", "toli":"cadastre_ln_toponymiques", "canu":"cadastre_pt_num", "cabu":"Batiments", "gept":"cadastre_points_generaux", "gepn":"cadastre_pol_gen", "inpt":"point", "geli":"cadastre_ln_generales", "inli":"cadastre_ln_informations", "topt":"point", } #connect to tables and create layers and correct urban styles for table in urb: style = urb[table] ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370') ft.default_style = style cat.save(ft) resource = ft.resource resource.title = options.alias+"_"+table resource.save() layer, created = Layer.objects.get_or_create(name=layerName, defaults={ "workspace": ws.name, "store": ds.name, "storeType": ds.resource_type, "typename": "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', #"owner": owner, "uuid": str(uuid.uuid4()), "bbox_x0": Decimal(resource.latlon_bbox[0]), "bbox_x1": Decimal(resource.latlon_bbox[1]), "bbox_y0": Decimal(resource.latlon_bbox[2]), "bbox_y1": Decimal(resource.latlon_bbox[3]) }) set_attributes(layer, overwrite=True) if created: layer.set_default_permissions()
def add_layer(shp): global ds_name, ws_name, appserver global instance cat = Catalog(appserver + '/rest') ws = cat.get_workspace(ws_name) ds = cat.get_store(ds_name, ws_name) print 'starting {0}...'.format(shp) components = dict((ext, shp + '.' + ext) for ext in ['shp', 'prj', 'shx', 'dbf']) cat.add_data_to_store(ds, shp, components, ws, True) print '{0} finished ...'.format(shp)
def _create_geoserver_geonode_layer(new_table, sld_type, title, the_user, permissions): # Create the Layer in GeoServer from the table try: cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", settings.OGC_SERVER['default']['USER'], settings.OGC_SERVER['default']['PASSWORD']) ds = cat.get_store("uploaded") # name of store in WFP-Geonode cat.publish_featuretype(new_table, ds, "EPSG:4326", srs="EPSG:4326") # changing the layer's title in the title set by the user resource = cat.get_resource(new_table, workspace="geonode") resource.title = title cat.save(resource) except Exception as e: msg = "Error creating GeoServer layer for %s: %s" % (new_table, str(e)) print msg return None, msg # Create the Layer in GeoNode from the GeoServer Layer try: link_to_sld = "{location}styles/{sld_type}".format( **{ 'location': settings.OGC_SERVER['default']['LOCATION'], 'sld_type': sld_type }) r = requests.get(link_to_sld) sld = r.text sld = sld.replace( "name_of_layer", new_table ) # "name_of_layer" is set in the predefined sld in geoserver (polygon_style, line_style, point_style) cat.create_style(new_table, sld, overwrite=True) style = cat.get_style(new_table) layer = cat.get_layer(new_table) layer.default_style = style cat.save(layer) gs_slurp(owner=the_user, filter=new_table) from geonode.base.models import ResourceBase layer = ResourceBase.objects.get(title=title) geoserver_post_save(layer, ResourceBase) # assign permissions for this layer permissions_dict = json.loads(permissions) # needs to be dictionary if permissions_dict is not None and len(permissions_dict.keys()) > 0: layer.set_permissions(permissions_dict) except Exception as e: msg = "Error creating GeoNode layer for %s: %s" % (new_table, str(e)) return None, msg
def handle_geoserver_layers(view_names, store_name, db_params, schema_name, logger=print): """Publish database views as GeoServer layers Parameters ---------- view_pairs: list An iterable of two-element tuples with the names of the ``coarse`` and ``detail`` database views that are to be published as GeoServer layers store_name: str Name of the GeoServer store to use. This store will be created in case it does not exist db_params: dict A mapping with parameters used for connecting to the database schema_name: str Name of the database schema where the views to publish reside logger: function, optional A function that is used to output information. Returns ------- list An iterable with the geoserver layers that have been published """ gs_catalog = Catalog( service_url=settings.OGC_SERVER["default"]["LOCATION"] + "rest", username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"]) logger("Retrieving geoserver workspace...") workspace = get_geoserver_workspace(gs_catalog) store = gs_catalog.get_store(store_name, workspace=workspace) if store is None: logger("Creating geoserver store...") store = get_postgis_store(gs_catalog, store_name, workspace, db_params, schema_name) layers = [] for view_name in view_names: logger("Adding {!r} as a geoserver layer...".format(view_name)) featuretype = gs_catalog.publish_featuretype(view_name, store, "EPSG:4326", srs="EPSG:4326") gs_catalog.save(featuretype) layers.append(featuretype) return layers
class GSAddMosaicGranule(BaseOperator): @apply_defaults def __init__(self, granule_abs_path, geoserver_rest_url, gs_user, gs_password, imagemosaic_storename, *args, **kwargs): self.granule_abs_path = granule_abs_path self.catalog = Catalog(geoserver_rest_url, gs_user, gs_password) self.store_name = imagemosaic_storename super(GSAddMosaicGranule, self).__init__(*args, **kwargs) def execute(self, context): log.info("GSAddMosaicGranule params list") log.info('Mosaic granule: %s', self.granule_abs_path) store = self.catalog.get_store(self.store_name) self.catalog.harvest_externalgranule('file://' + self.granule_abs_path, store)
def handle_geoserver_layer(table_name, store_name, db_params, schema_name, default_style_name="", logger=print): """Publish a database table/view as a GeoServer layer Parameters ---------- store_name: str Name of the GeoServer store to use. This store will be created in case it does not exist db_params: dict A mapping with parameters used for connecting to the database schema_name: str Name of the database schema where the views to publish reside logger: function, optional A function that is used to output information. Returns ------- list An iterable with the geoserver layers that have been published """ logger("inside handle_geoserver_layer: {}".format(locals())) gs_catalog = Catalog( service_url=settings.OGC_SERVER["default"]["LOCATION"] + "rest", username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"] ) logger("Retrieving geoserver workspace...") workspace = get_geoserver_workspace(gs_catalog) store = gs_catalog.get_store(store_name, workspace=workspace) if store is None: logger("Creating geoserver store...") store = get_postgis_store(gs_catalog, store_name, workspace, db_params, schema_name) logger("Adding {!r} as a geoserver layer...".format(table_name)) featuretype = gs_catalog.publish_featuretype( table_name, store, "EPSG:4326", srs="EPSG:4326") logger("dir(featuretype): {}".format(dir(featuretype))) layer = gs_catalog.get_layer(featuretype.name) if default_style_name != "": logger("Setting default style for layer...") layer._set_default_style(default_style_name) gs_catalog.save(featuretype) gs_catalog.save(layer) return featuretype
def pre_delete_service(instance, sender, **kwargs): for layer in instance.layer_set.all(): layer.delete() # if instance.method == 'H': # gn = Layer.objects.gn_catalog # gn.control_harvesting_task('stop', [instance.external_id]) # gn.control_harvesting_task('remove', [instance.external_id]) if instance.method == 'C': try: _user = settings.OGC_SERVER['default']['USER'] _password = settings.OGC_SERVER['default']['PASSWORD'] gs = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user , _password) cascade_store = gs.get_store(instance.name, settings.CASCADE_WORKSPACE) gs.delete(cascade_store, recurse=True) except FailedRequestError: logger.error("Could not delete cascading WMS Store for %s - maybe already gone" % instance.name)
def createLayer(): cat = Catalog('http://localhost:8081/geoserver/rest/', 'admin', 'geoserver') store = cat.get_store('test') geom = JDBCVirtualTableGeometry('newgeom', 'LineString', '4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = 'select ST_MakeLine(wkb_geometry ORDER BY waypoint) As newgeom, assetid, runtime from waypoints group by assetid,runtime' keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) ft = cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt)
def pre_delete_service(instance, sender, **kwargs): for layer in instance.layer_set.all(): layer.delete() # if instance.method == 'H': # gn = Layer.objects.gn_catalog # gn.control_harvesting_task('stop', [instance.external_id]) # gn.control_harvesting_task('remove', [instance.external_id]) if instance.method == 'C': try: _user = settings.OGC_SERVER['default']['USER'] _password = settings.OGC_SERVER['default']['PASSWORD'] gs = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) cascade_store = gs.get_store( instance.name, settings.CASCADE_WORKSPACE) gs.delete(cascade_store, recurse=True) except FailedRequestError: logger.error( "Could not delete cascading WMS Store for %s - maybe already gone" % instance.name)
class GSAddMosaicGranule(BaseOperator): @apply_defaults def __init__(self, get_inputs_from, geoserver_rest_url, gs_user, gs_password, store_name, workspace=None, *args, **kwargs): self.get_inputs_from = get_inputs_from self.catalog = Catalog(geoserver_rest_url, gs_user, gs_password) self.store_name = store_name self.workspace = workspace log.info( '--------------------GeoServer_PLUGIN Add granule------------') super(GSAddMosaicGranule, self).__init__(*args, **kwargs) def execute(self, context): input_paths = context["task_instance"].xcom_pull(self.get_inputs_from, key=XCOM_RETURN_KEY) if input_paths is None: log.info("Nothing to process") return None # If message from XCom is a string with single file path, turn it into a string if isinstance(input_paths, six.string_types): input_paths = [input_paths] for granule in input_paths: store = self.catalog.get_store(self.store_name, workspace=self.workspace) if store is None: log.info("GeoServer Store {} is not accessible!".format( self.store_name)) return None granule = 'file://' + granule log.info('Mosaic granule: %s', granule) self.catalog.add_granule(granule, store, workspace=self.workspace)
class GSAddMosaicGranule(BaseOperator): @apply_defaults def __init__(self, geoserver_rest_url, gs_user, gs_password, imagemosaic_storename, mosaic_path, index, *args, **kwargs): self.catalog = Catalog(geoserver_rest_url, gs_user, gs_password) self.store_name = imagemosaic_storename self.mosaic_path = mosaic_path self.index = index log.info('--------------------GDAL_PLUGIN Add granule------------') super(GSAddMosaicGranule, self).__init__(*args, **kwargs) def execute(self, context): task_instance = context['task_instance'] granule = task_instance.xcom_pull('rsync_' + str(self.index), key=xk.GRANULE_TO_UPLOAD_PREFIX + str(self.index)) log.info("GSAddMosaicGranule params list") log.info('Mosaic granule: %s', granule) store = self.catalog.get_store(self.store_name) granule = 'file://' + self.mosaic_path + '/' + granule log.info(granule) self.catalog.harvest_externalgranule(granule, store)
def deleteGeoserverStore(storeName, workspace=workspace, purge=None, recurse=True): """ Delete Geoserver Data Store. Args: storeName Kwargs (Default): workspace(geocolorado) purge(None) - if purge is True. Data files will be deleted. recurse (True) - Store and all metadata items will be deleted. """ cat = Catalog("{0}/rest/".format(geoserver_connection), geoserver_username, geoserver_password) ws = cat.get_workspace(workspace) ds = cat.get_store(storeName, workspace=ws) cat.delete(ds, purge=purge, recurse=recurse) msg = "metadata and data files removed." if purge else "only metadata items removed." return "DataStore: {0} deleted from geoServer with {1}".format( storeName, msg)
def remove_geoserver_layer(self, storename, layername): self.logger.debug("start removing the geoserver layer") last_charactor = self.geoServer[-1] if last_charactor == '/': geoserver_rest = self.geoServer + 'rest' else: geoserver_rest = self.geoServer + '/rest' cat = Catalog(geoserver_rest, username=self.gs_username, password=self.gs_password) # worksp = cat.get_workspace(gs_workspace) store = cat.get_store(storename) self.logger.debug("store name %s" % store) layer = cat.get_layer(layername) self.logger.debug("layer name %s" % layer) try: self.logger.debug("deleting the layer...") cat.delete(layer) cat.reload() self.logger.debug("deleting the store...") cat.delete(store) cat.reload except Exception: self.logger.error("Failed to remove from geoserver")
def publishGeoserver(appdef, progress): viewCrs = appdef["Settings"]["App view CRS"] usesGeoServer = False for applayer in appdef["Layers"]: if applayer.method != METHOD_FILE: if applayer.layer.type() == applayer.layer.VectorLayer and applayer.layer.providerType().lower() != "wfs": usesGeoServer = True if not usesGeoServer: return progress.setText("Publishing to GeoServer") progress.setProgress(0) geoserverUrl = appdef["Deploy"]["GeoServer url"] + "/rest" geoserverPassword = appdef["Deploy"]["GeoServer password"] geoserverUsername = appdef["Deploy"]["GeoServer username"] workspaceName = appdef["Deploy"]["GeoServer workspace"] dsName = "ds_" + workspaceName host = appdef["Deploy"]["PostGIS host"] port = appdef["Deploy"]["PostGIS port"] postgisUsername = appdef["Deploy"]["PostGIS username"] postgisPassword = appdef["Deploy"]["PostGIS password"] database = appdef["Deploy"]["PostGIS database"] schema = appdef["Deploy"]["PostGIS schema"] catalog = Catalog(geoserverUrl, geoserverUsername, geoserverPassword) workspace = catalog.get_workspace(workspaceName) if workspace is None: workspace = catalog.create_workspace(workspaceName, workspaceName) try: store = catalog.get_store(dsName, workspace) resources = store.get_resources() for resource in resources: layers = catalog.get_layers(resource) for layer in layers: catalog.delete(layer) catalog.delete(resource) catalog.delete(store) except Exception: pass try: store = catalog.get_store(dsName, workspace) except FailedRequestError: store = None for i, applayer in enumerate(appdef["Layers"]): layer = applayer.layer if applayer.method != METHOD_FILE and applayer.method != METHOD_DIRECT: name = safeName(layer.name()) sld, icons = getGsCompatibleSld(layer) if sld is not None: catalog.create_style(name, sld, True) uploadIcons(icons, geoserverUsername, geoserverPassword, catalog.gs_base_url) if layer.type() == layer.VectorLayer: if applayer.method == METHOD_WFS_POSTGIS or applayer.method == METHOD_WMS_POSTGIS: if store is None: store = catalog.create_datastore(dsName, workspace) store.connection_parameters.update( host=host, port=str(port), database=database, user=postgisUsername, schema=schema, passwd=postgisPassword, dbtype="postgis") catalog.save(store) catalog.publish_featuretype(name, store, layer.crs().authid()) else: path = getDataFromLayer(layer, viewCrs) catalog.create_featurestore(name, path, workspace=workspace, overwrite=True) gslayer = catalog.get_layer(name) r = gslayer.resource r.dirty['srs'] = viewCrs catalog.save(r) elif layer.type() == layer.RasterLayer: path = getDataFromLayer(layer, viewCrs) catalog.create_coveragestore(name, path, workspace=workspace, overwrite=True) if sld is not None: publishing = catalog.get_layer(name) publishing.default_style = catalog.get_style(name) catalog.save(publishing) progress.setProgress(int((i+1)*100.0/len(appdef["Layers"])))
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. try: if lyr: self.cat.delete(lyr) self.cat.delete(lyr.resource) if ds: self.cat.delete(ds) except: pass def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store(ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) @drop_table('import2') def testVirtualTables(self): ds = self.cat.create_datastore("gsconfig_import_test2") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test2") self.cat.add_data_to_store(ds, "import2", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) store = self.cat.get_store("gsconfig_import_test2") geom = JDBCVirtualTableGeometry('the_geom','MultiPolygon','4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = "select * from import2 where 'STATE_NAME' = 'Illinois'" keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) ft = self.cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt) # DISABLED; this test works only in the very particular case # "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR # def testCoverageStoreCreate(self): # ds = self.cat.create_coveragestore2("coverage_gsconfig") # ds.data_url = "file:test/data/mytiff.tiff" # self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) # Testing projection and projection policy changes changed_layer.projection = "EPSG:900913" changed_layer.projection_policy = "REPROJECT_TO_DECLARED" self.cat.save(changed_layer) self.cat._cache.clear() layer = self.cat.get_layer(changed_layer.name) self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy) self.assertEqual(layer.resource.projection, changed_layer.projection) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf) ) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf) ) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf) ) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff) ) ft_ext = self.cat.create_coveragestore_external_geotiff("Pk50095_ext", 'file:test/data/Pk50095.tif', sf) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = { 'title': 'Not the original attribution', 'width': '123', 'height': '321', 'href': 'http://www.georchestra.org', 'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg', 'type': 'image/jpeg' } # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # check count before tests (upload) count = len(self.cat.get_styles()) # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assert_(f is not None) self.assertEqual(f.name, fred.name) # compare count after upload self.assertEqual(count +1, len(self.cat.get_styles())) # attempt creating a new style without "title" self.cat.create_style("notitle", open("test/notitle.sld").read()) notitle = self.cat.get_style("notitle") self.assertEqual(None, notitle.sld_title) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assert_(jed is None) jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assert_(self.cat.get_style("jed", workspace="topp") is None) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assert_(f is not None) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assert_(ned is not None) self.assert_(zed is not None) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testImageMosaic(self): """ Test case for Issue #110 """ # testing the mosaic creation name = 'cea_mosaic' data = open('test/data/mosaic/cea.zip', 'rb') self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assert_(resource is not None) # delete granule from mosaic coverage = name store = self.cat.get_store(name) granule_id = name + '.1' self.cat.mosaic_delete_granule(coverage, store, granule_id) def testTimeDimension(self): sf = self.cat.get_workspace("sf") files = shapefile_and_friends(os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer('boxes_with_end_date').resource # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time':timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time' : timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time' : timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute)
ds_name = 'ne_pg' lg_name = "ne_pg" cat = Catalog(appserver + '/rest') # check if workspace exists, bail if it does (safter than deleting it) if any(ws.name == ws_name for ws in cat.get_workspaces()): print 'workspace already exists...' ws = cat.get_workspace(ws_name) else: print 'creating workspace...' ws = cat.create_workspace(ws_name, 'http://www.naturalearth.org') if any(ds.workspace.name == ws.name and ds.name == ds_name for ds in cat.get_stores()): print 'datastore already exists...' ds = cat.get_store(ds_name, ws_name) else: print 'creating datastore' ds = cat.create_datastore(ds_name, ws.name) ds.connection_parameters.update( host=dbserver, port='5432', database='ne_tmp', user='******', password='******', dbtype='postgis') cat.save(ds) ds = cat.get_store(ds_name, ws_name) layers = [] styles = []
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testAbout(self): about_html = self.cat.about() self.assertTrue( '<html xmlns="http://www.w3.org/1999/xhtml"' in str(about_html)) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d+(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertTrue(pat.match('2.10.1')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) self.assertEqual("topp", self.cat.get_workspaces("topp")[-1].name) self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde']))) self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde'))) def testStores(self): self.assertEqual(0, len(self.cat.get_stores("nonexistentstore"))) topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(workspace=topp))) self.assertEqual(2, len(self.cat.get_stores(workspace=sf))) self.assertEqual(2, len(self.cat.get_stores(workspace='sf'))) self.assertEqual( 2, len(self.cat.get_stores(names='states_shapefile, sfdem'))) self.assertEqual( 2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem']))) self.assertEqual("sfdem", self.cat.get_stores("sfdem")[-1].name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", "topp").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testResourcesUpdate(self): res_dest = self.cat.get_resources() count = 0 for rd in res_dest: # only wms layers if rd.resource_type != "wmsLayer": continue # looking for same name ro = self.cat.get_resource(rd.name) if ro is not None: rd.title = ro.title rd.abstract = ro.abstract rd.keywords = ro.keywords rd.projection = ro.projection rd.native_bbox = ro.native_bbox rd.latlon_bbox = ro.latlon_bbox rd.projection_policy = ro.projection_policy rd.enabled = ro.enabled rd.advertised = ro.advertised rd.metadata_links = ro.metadata_links or None self.cat.save(rd) self.cat.reload() # print "Updated layer: " + rd.name count += 1 # print "Total updated layers: " + str(count) def testLayers(self): expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assertTrue(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assertEqual("states", states.name) self.assertIsInstance(states.resource, ResourceInfo) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assertTrue(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assertEqual("tasmania", tas.name) self.assertIsInstance(tas, LayerGroup) self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) # Try to create a new Layer Group into the "topp" workspace self.assertIsNotNone(self.cat.get_workspace("topp")) tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace="topp") self.cat.save(tas2) self.assertIsNone(self.cat.get_layergroup("tasmania_reloaded")) self.assertIsNotNone( self.cat.get_layergroup("tasmania_reloaded", "topp")) tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp") self.assertEqual("tasmania_reloaded", tas2.name) self.assertIsInstance(tas2, LayerGroup) self.assertEqual(tas2.workspace, "topp", tas2.workspace) self.assertEqual(tas2.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas2.layers) self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles) def testStyles(self): self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assertIsNone(self.cat.get_style('non-existing-style')) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") self.assertEqual( self.cat.get_store(workspace="best workspace ever", name="best store ever"), None) self.cat.get_layer("best layer ever") self.cat.get_layergroup("best layergroup ever") def testUnicodeUrl(self): """ Tests that the geoserver.support.url function support unicode strings. """ # Test the url function with unicode seg = [ 'workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml' ] u = url(base=self.cat.service_url, seg=seg) self.assertEqual( u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml") # Test the url function with normal string seg = [ 'workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml' ] u = url(base=self.cat.service_url, seg=seg) self.assertEqual( u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
def calccosts(cf, layer_name, json_matrix): new_layer = "classroads_{}".format(int(1000000 * time.time())) # calccost function calculates recalcutes the visualisation caterogies engine = create_engine( "postgresql+psycopg2://" + cf.get("PostGis", "user") + ":" + cf.get("PostGis", "pass") + "@" + cf.get("PostGis", "host") + ":" + str(cf.get("PostGis", "port")) + "/" + cf.get("PostGis", "db"), strategy="threadlocal", pool_pre_ping=True, ) # try: # TODO make the ra2ce schema name as a parameter (input to the service) strSql = """create table temp.{new_layer} as select geom, societal_class, repair_class, 0 as visclass from public.{layer_name}_priority;""".format( new_layer=new_layer, layer_name=layer_name ) ressql = engine.execute(strSql) print("create temp table", ressql) # except exc.DBAPIError as e: # an exception is raised, Connection is invalidated. # if e.connection_invalidated: # logging.info("Connection was invalidated!") data = json.loads(json_matrix) values = data["values"] for societalIndex in range(len(values)): for repairIndex in range(len(values[societalIndex])): strSql = """update temp.{new_layer} set visclass = {val} where societal_class = {s} and repair_class = {r};""".format( new_layer=new_layer, val=values[societalIndex][repairIndex], s=societalIndex + 1, r=repairIndex + 1, ) ressql = engine.execute(strSql) ressql.close() # publish the layer in geoserver cat = Catalog( "{}/rest/".format(cf.get("GeoServer", "host")), username=cf.get("GeoServer", "user"), password=cf.get("GeoServer", "pass"), ) print("cat", cat) datastore = cat.get_store("temp") ft = cat.publish_featuretype( new_layer, datastore, native_crs="EPSG:4326", jdbc_virtual_table="" ) cat.save(ft) layer = cat.get_layer(new_layer) print("layer", layer) # check if shapefile is line or point # TODO check styles with Gerrit style = "ra2ce" # set style and save layer # layer._set_default_style(style) layer.default_style = style cat.save(layer) res = writeOutput(cf=cf, wmslayer="ra2ce-dr:{}".format(new_layer), defstyle=style) # time.sleep(2) return res
class GeoserverManager(): gs_master = None gs_slaves = [] def __init__(self, config, disable_ssl_certificate_validation=False): # settings config = config["settings"]["geoserver"] # log.info(config["geoserver_master"]) # log.info(config["password"]) # log.info(config["username"]) self.config = config if "geoserver_master" in config: print config["geoserver_master"] self.gs_master = Catalog(config["geoserver_master"], self.config["username"], self.config["password"]) else: raise Exception('config["geoserver_master"] has to be mapped to a running Geoserver') if "geoserver_slaves" in config: for gs_slave in config["geoserver_slaves"]: self.gs_slaves.append(Catalog(gs_slave, self.config["username"], self.config["password"])) def _initialize_geoserver(self, geoserver): geoserver.username = self.config["username"] geoserver.password = self.config["password"] def publish_coveragestore(self, path, data, overwrite=False, reload_gs_slaves=True): log.info(data) workspace = self.gs_master.get_default_workspace() if "workspace" not in data else self.check_workspace(data["workspace"], False) if "layerName" not in data: raise Exception("No layerName found in the metadata") name = data["layerName"] # publish self.gs_master.create_coveragestore(name, path, workspace, overwrite) # setting default style if "defaultStyle" in data: self.set_style(name, workspace.name, data["defaultStyle"], False) # reload geoserver slaves if reload_gs_slaves: self.reload_gs_slaves() return True def publish_postgis_table(self, data, reload_gs_slaves=True, native_crs=None, srs=None, overwrite=False): try: '''Publish a featuretype from data in an existing store''' # @todo native_srs doesn't seem to get detected, even when in the DB # metadata (at least for postgis in geometry_columns) and then there # will be a misconfigured layer name = data["layerName"] title = data["title"] if "title" in data else name store = self.gs_master.get_store(data["store"]) # #if native_crs is None: raise ValueError("must specify native_crs") #srs = srs or native_crs feature_type = FeatureType(self, store.workspace, store, name) # because name is the in FeatureType base class, work around that # and hack in these others that don't have xml properties feature_type.dirty['name'] = name # feature_type.dirty['srs'] = srs # feature_type.dirty['nativeCRS'] = native_crs feature_type.enabled = True feature_type.title = title headers = { "Content-type": "application/xml", "Accept": "application/xml" } headers, response = self.gs_master.http.request(store.resource_url, "POST", feature_type.message(), headers) if headers.status != 201: raise Exception(response, headers.status) # defaultStyle if "defaultStyle" in data: # TODO: test it self.set_style(name, store.workspace.name, data["defaultStyle"], False) # reload geoservers if reload_gs_slaves: self.reload_gs_slaves() return feature_type except Exception, (response, status): raise Exception(response, status)
def synchronizeLayers(self): ''' Synchronizes layers/rules in the postgresql database, Geoserver, and Geofence. All three will match postgresql database. ''' to_return = True permanent_gs_layers = ['rapid_geoview_layers','rapid_feature','rapid_geoview'] rapid_layers = {lyr.uid for lyr in DataLayer.objects.all() if lyr.feature_set.count() > 0} cat = Catalog(GEOSERVER_REST_ENDPOINT) store = cat.get_store('RapidData') gs_layers = {lyr.name for lyr in store.get_resources() if lyr.name not in permanent_gs_layers} gf_rules_ids = {(rule.find('layer').text, int(rule.find('id').text)) for rule in ET.fromstring(getGeofenceRules('admin').content)[:-1]} gf_rules = {rule[0] for rule in gf_rules_ids} gs_layers_to_remove = gs_layers.difference(rapid_layers) gs_layers_to_add = rapid_layers.difference(gs_layers) gf_rules_to_remove = gf_rules.difference(rapid_layers) gf_rules_to_add = rapid_layers.difference(gf_rules) if len(gs_layers_to_add) == 0 and len(gs_layers_to_remove) == 0: if len(gf_rules_to_add) == 0 and len(gf_rules_to_remove) == 0: return to_return elif len(gs_layers_to_remove) > 0: for lyr_uid in gs_layers_to_remove: print "Removing Geoserver layer {0}. . .".format(lyr_uid) response = self.removeFeatureType(lyr_uid) if response.status_code == 200: print "Success!" else: print "Failed to remove layer {0}".format(lyr_uid) to_return = False print response.content elif len(gs_layers_to_add) > 0: for uid in gs_layers_to_add: print "Adding FeatureType to Geoserver: {0}".format(uid) xml = self.createFeatureTypeFromUid(uid) response = self.sendFeatureType(xml) if response.status_code == 200: print "Success!" else: print "Failed to add FeatureType {0}".format(uid) to_return = False print response.content if len(gf_rules_to_add) == 0 and len(gf_rules_to_remove) == 0: return to_return elif len(gf_rules_to_remove) > 0: gf_ids_to_remove = [rule for rule in gf_rules_ids if rule[0] in gf_rules_to_remove] for rule in gf_ids_to_remove: print "Removing Geofence rule for layer {0}".format(rule[0]) response = removeGeofenceRule(rule[1]) if response.status_code == 200: print "Success!" else: print "Failed to remove layer {0}".format(lyr_uid) to_return = false print response.content elif len(gf_rules_to_add) > 0: for rule in gf_rules_to_add: response = addGeofenceRule(layer=rule) if response.status_code == 200: print "Success!" else: print "Failed to add Geofence rule for layer {0}".format(rule) to_return = False print response.content return to_return
The GeoServer USERNAME and PASSWORD *are not* the ones configured on GeoNode. It is mandatory to create a GeoServer manager user (by default admin/geoserver) with WRITE (or ADMIN) rights and BASIC AUTH access enabled. """ url = "http://*****:*****@time [" + json.dumps(feature["properties"]["time"]) + "]" granule_id = feature["id"] gs_catalog.mosaic_delete_granule(coverages['coverages']['coverage'][0]['name'], store, granule_id) print "Deleting file [" + json.dumps(feature["properties"]["location"]) + "]" try: os.remove(feature["properties"]["location"]) except Exception as e: print "Could not delete the GeoTIFF Granule [" + str(feature["properties"]["location"]) + "]: " + str(e)
#!/usr/bin/env python ''' gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . ''' __author__ = "David Winslow" __copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") ds = cat.create_datastore(name) ds.connection_parameters.update( host="localhost", port="5432", database="gis", user="******", passwd="", dbtype="postgis") cat.save(ds) ds = cat.get_store(name) components = \ dict((ext, "myfile." + ext) for ext in ["shp", "prj", "shx", "dbf"]) cat.add_data_to_store(ds, "mylayer", components)
def export(uid): """This function is used to show and handle the export options and uses the GeoServer REST API to export results. This function uses the GeoServer configuration client library by boundlessgeo, see: https://github.com/boundlessgeo/gsconfig To export the results, the GeoServer application has to be running and should be correctly set up to have access to the database and the styles should already be imported, see geoserver styles folder. GET request: Renders and returns a site showing export options. POST request: Gets the export options from the user and exports the results if they are defined in the options. The export is made by using GeoServer configuration client library. After the export, the newly created layers should be visible in the GeoServer web interface and the WM(T)S links should also work. These links can then be used to display the WM(T)S layers in JOSM or a gis. Remarks: This function will be redesigned in future versions """ uid = uid.encode("ISO-8859-1") if request.method == "POST": dm = DevMap.query.filter_by(uid=uid).first() if dm.owner == current_user or dm.owner == User.query.filter_by(username="******").first(): title = request.form["title"] listedmap = False if "listed" in request.form: listedmap = True dm.title = title dm.listed = listedmap cat = Catalog(gs_url + "rest") cat.username = gs_user cat.password = gs_password ws = None try: ws = cat.get_workspace(gs_workspace) except socket.error, e: db.session.add(dm) db.session.commit() return render_template("export.html", uid=uid, error=e, dm=dm) st = cat.get_store(gs_store, ws) if "maxdevgrid" in request.form: feattype = "odf_" + uid + "_maxdevgrid" if dm.wmsmaxdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":maxdevgrid") cat.save(l) dm.wmsmaxdevgrid = True else: dm.wmsmaxdevgrid = True if "posdevlines" in request.form: feattype = "odf_" + uid + "_posdevlines" if dm.wmsposdevlines: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":posdevlines") cat.save(l) dm.wmsposdevlines = True else: dm.wmsposdevlines = False if "absdevgrid" in request.form: feattype = "odf_" + uid + "_absdevgrid" if dm.wmsabsdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":absdevgrid") cat.save(l) dm.wmsabsdevgrid = True else: dm.wmsabsdevgrid = False if "matchingrategrid" in request.form: feattype = "odf_" + uid + "_matchingrategrid" if dm.wmsmatchingrategrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":matchingrategrid") cat.save(l) dm.wmsmatchingrategrid = True else: dm.wmsmatchingrategrid = False if "unmatchedref" in request.form: feattype = "odf_" + uid + "_unmatchedref" if dm.wmsunmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.unmatchedref = True else: dm.unmatchedref = False if "unmatchedosm" in request.form: feattype = "odf_" + uid + "_unmatchedosm" if dm.unmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":OSMLines") cat.save(l) dm.wmsunmatchedosm = True else: dm.wmsunmatchedosm = False if "matchedref" in request.form: feattype = "odf_" + uid + "_matchedref" if dm.wmsmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.wmsmatchedref = True else: dm.wmsmatchedref = False if "matchedosm" in request.form: feattype = "odf_" + uid + "_matchedosm" if dm.wmsmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":OSMLines") cat.save(l) dm.wmsmatchedosm = True else: dm.wmsmatchedosm = False if "minlevenshtein" in request.form: feattype = "odf_" + uid + "_minlevenshtein" if dm.wmsminlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.wmsminlevenshtein = True else: dm.wmsminlevenshtein = False if "maxlevenshtein" in request.form: feattype = "odf_" + uid + "_maxlevenshtein" if dm.wmsmaxlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") cat.save(ft) l = cat.get_layer(feattype) l._set_default_style(gs_workspace + ":ReferenceLines") cat.save(l) dm.wmsmaxlevenshtein = True else: dm.wmsmaxlevenshtein = False db.session.add(dm) db.session.commit() return render_template("finished.html", uid=uid) else: return render_template("export.html", uid=uid, error="No User", dm=dm)
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) self.gs_version = self.cat.get_short_version() def testGSVersion(self): version = self.cat.get_version() pat = re.compile('\d\.\d+') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspaces(names="topp")[-1].name) self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde']))) self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde'))) self.assertEqual("topp", self.cat.get_workspace("topp").name) self.assertIsNone(self.cat.get_workspace("blahblah-")) def testStores(self): self.assertEqual(0, len(self.cat.get_stores(names="nonexistentstore"))) topp = self.cat.get_workspaces("topp")[0] sf = self.cat.get_workspaces("sf")[0] self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(workspaces=topp))) self.assertEqual(2, len(self.cat.get_stores(workspaces=sf))) self.assertEqual(2, len(self.cat.get_stores(workspaces='sf'))) self.assertEqual(2, len(self.cat.get_stores(names='states_shapefile, sfdem'))) self.assertEqual(2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem']))) self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0].name) self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile")[0].name) self.assertEqual("sfdem", self.cat.get_stores(names="sfdem", workspaces=sf.name)[0].name) self.assertEqual("sfdem", self.cat.get_stores(names="sfdem")[0].name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) self.assertIsNone(self.cat.get_store("blah+blah-")) def testResources(self): topp = self.cat.get_workspaces("topp")[0] sf = self.cat.get_workspaces("sf")[0] states = self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0] sfdem = self.cat.get_stores(names="sfdem", workspaces=sf.name)[0] self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(2, len(self.cat.get_resources(stores=[states.name, sfdem.name]))) self.assertEqual(11, len(self.cat.get_resources(workspaces=[topp.name, sf.name]))) self.assertEqual("states", self.cat.get_resources(names="states", stores=states.name)[0].name) self.assertEqual("states", self.cat.get_resources(names="states", workspaces=topp.name)[0].name) self.assertEqual("states", self.cat.get_resources(names="states")[0].name) self.assertEqual("states", self.cat.get_resource("states").name) self.assertIsNone(self.cat.get_resource("blah+1blah-2")) states = self.cat.get_resources(names="states")[0] fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", stores=sfdem.name)[0].name) self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", workspaces=sf.name)[0].name) self.assertEqual("sfdem", self.cat.get_resources(names="sfdem")[0].name) def testResourcesUpdate(self): res_dest = self.cat.get_resources() count = 0 for rd in res_dest: # only wms layers if rd.resource_type != "wmsLayer": continue # looking for same name ro = self.cat.get_resources(names=rd.name) if ro is not None: rd.title = ro.title rd.abstract = ro.abstract rd.keywords = ro.keywords rd.projection = ro.projection rd.native_bbox = ro.native_bbox rd.latlon_bbox = ro.latlon_bbox rd.projection_policy = ro.projection_policy rd.enabled = ro.enabled rd.advertised = ro.advertised rd.metadata_links = ro.metadata_links or None self.cat.save(rd) self.cat.reload() count += 1 def testLayers(self): if self.gs_version >= "2.13": expected = set([ 'sf:roads', 'sf:sfdem', 'nurc:mosaic', 'tiger:giant_polygon', 'sf:bugsites', 'topp:states', 'sf:streams', 'tiger:poly_landmarks', 'tiger:poi', 'topp:tasmania_water_bodies', 'tiger:tiger_roads', 'topp:tasmania_roads', 'nurc:Pk50095', 'topp:tasmania_cities', 'nurc:Img_Sample', 'sf:restricted', 'nurc:Arc_Sample', 'sf:archsites', 'topp:tasmania_state_boundaries' ]) else: expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups(names=["tasmania", "tiger-ny", "spearfish"])) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroups(names="tasmania")[0] self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) if self.gs_version >= "2.13": self.assertEqual(tas.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads', 'topp:tasmania_cities' ], tas.layers) else: self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) # Try to create a new Layer Group into the "topp" workspace self.assert_(self.cat.get_workspaces("topp")[0] is not None) tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp") self.cat.save(tas2) self.assertEqual(1, len(self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp"))) tas2 = self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")[0] self.assert_("tasmania_reloaded", tas2.name) self.assert_(isinstance(tas2, LayerGroup)) self.assertEqual(tas2.workspace, "topp", tas2.workspace) if self.gs_version >= "2.13": self.assertEqual(tas2.layers, [ 'topp:tasmania_state_boundaries', 'topp:tasmania_water_bodies', 'topp:tasmania_roads', 'topp:tasmania_cities' ], tas2.layers) else: self.assertEqual(tas2.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas2.layers) self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles) def testStyles(self): self.assertEqual("population", self.cat.get_styles("population")[0].name) self.assertEqual("popshade.sld", self.cat.get_styles("population")[0].filename) self.assertEqual("population", self.cat.get_styles("population")[0].sld_name) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assertIsNone(self.cat.get_style("blah+#5blah-")) self.assertEqual(0, len(self.cat.get_styles('non-existing-style'))) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_styles("best style ever") self.cat.get_workspaces("best workspace ever") self.assertEqual(0, len(self.cat.get_stores(workspaces="best workspace ever", names="best store ever"))) self.cat.get_layer("best layer ever") self.cat.get_layergroups("best layergroup ever") def testUnicodeUrl(self): """ Tests that the geoserver.support.url function support unicode strings. """ # Test the url function with unicode seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml'] u = build_url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml") # Test the url function with normal string seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'] u = build_url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
def delete(uid): """A function to delete a deviation map, all its tables, files and GeoServer layers. GET request: Renders and returns a site showing delete options. POST request: Gets delete options chosen by user and uses them to delete the chosen parts of the deviation map. To delete GeoServer layers the GeoServer configuration client library is used. """ uid = uid.encode("ISO-8859-1") if request.method == "POST": dm = DevMap.query.filter_by(uid=uid).first() if ( current_user.is_authenticated() and dm.owner == current_user or dm.owner == User.query.filter_by(username="******").first() ): if ( dm.wmsposdevlines or dm.wmsmaxdevgrid or dm.wmsabsdevgrid or dm.wmsmatchingrategrid or dm.wmsunmatchedref or dm.wmsunmatchedosm or dm.wmsmatchedref or dm.wmsmatchedosm or dm.wmsminlevenshtein or dm.wmsmaxlevenshtein ): cat = Catalog(gs_url + "rest") cat.username = gs_user cat.password = gs_password ws = None try: ws = cat.get_workspace(gs_workspace) except socket.error, e: detail = "GeoServer is not available. Make sure that it is running and the connection is ok." return render_template("error.html", err=e, detail=detail) st = cat.get_store(gs_store, ws) if "deletemaxdevgrid" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_maxdevgrid" if dm.wmsmaxdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmaxdevgrid = False if "deleteposdevlines" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_posdevlines" if dm.wmsposdevlines: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsposdevlines = False if "deleteabsdevgrid" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_absdevgrid" if dm.wmsabsdevgrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) ft = cat.publish_featuretype(feattype, st, "EPSG:4326") if ft is not None: cat.delete(ft) dm.wmsabsdevgrid = False if "deletematchingrategrid" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_matchingrategrid" if dm.wmsmatchingrategrid: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmatchingrategrid = False if "deleteunmatchedref" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_unmatchedref" if dm.wmsunmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.deleteunmatchedref = False if "deleteunmatchedosm" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_unmatchedosm" if dm.wmsunmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsunmatchedosm = False if "deletematchedref" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_matchedref" if dm.wmsmatchedref: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmatchedref = False if "deletematchedosm" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_matchedosm" if dm.wmsmatchedosm: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmatchedosm = False if "deleteminlevenshtein" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_minlevenshtein" if dm.wmsminlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsminlevenshtein = False if "deletemaxlevenshtein" in request.form or "deleteall" in request.form: feattype = "odf_" + uid + "_maxlevenshtein" if dm.wmsmaxlevenshtein: l = cat.get_layer(feattype) if l is not None: cat.delete(l) dm.wmsmaxlevenshtein = False if "deleteall" in request.form: folder = secure_filename(uid) folder = os.path.join(app.config["UPLOAD_FOLDER"], folder) shutil.rmtree(folder, True) db.engine.execute("drop table if exists odf_" + uid + "_ref") db.engine.execute("drop table if exists odf_" + uid + "_ref_presplitted") db.engine.execute("drop table if exists odf_" + uid + "_ref_splitted") db.engine.execute("drop table if exists odf_" + uid + "_found") db.engine.execute("drop table if exists odf_" + uid + "_ref_junctions") db.engine.execute("drop table if exists odf_" + uid + "_ref_points") db.engine.execute("drop table if exists odf_" + uid + "_ref_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_ref_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted") db.engine.execute("drop table if exists odf_" + uid + "_osm_splitted") db.engine.execute("drop table if exists odf_" + uid + "_osm_junctions") db.engine.execute("drop table if exists odf_" + uid + "_osm_points") db.engine.execute("drop table if exists odf_" + uid + "_osm_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_unmatchedref;") db.engine.execute("drop table if exists odf_" + uid + "_unmatchedosm;") db.engine.execute("drop table if exists odf_" + uid + "_minlevenshtein;") db.engine.execute("drop table if exists odf_" + uid + "_maxlevenshtein;") db.engine.execute("drop table if exists odf_" + uid + "_grid;") db.engine.execute("drop table if exists odf_" + uid + "_maxdevgrid;") db.engine.execute("drop table if exists odf_" + uid + "_matchingrategrid;") db.engine.execute("drop table if exists odf_" + uid + "_deviationlines") db.engine.execute("drop table if exists odf_" + uid + "_junction_deviationlines") if DEBUG: db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_junctions") db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_points") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_cutcheckpoints") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_cutpoints") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_junction_devvec") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_junctions") db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_points") db.engine.execute("drop table if exists odf_" + uid + "_result") if "deleteall" not in request.form: db.session.add(dm) db.session.commit() return render_template("delete.html", uid=uid, dm=dm, error=None) else: db.session.delete(dm) db.session.commit() return redirect(url_for("basic.index")) else: return render_template("error.html", err="You are not allowed to delete this map!")
class GeoGigUploaderBase(ImportHelper): def __init__(self, *args, **kwargs): super(GeoGigUploaderBase, self).__init__(*args, **kwargs) setUpModule() # this isn't available when being used in other module def setUp(self): self.admin_user = self.create_user('admin', 'admin', is_superuser=True) self.non_admin_user = self.create_user('non_admin', 'non_admin') self.catalog = Catalog( ogc_server_settings.internal_rest, *ogc_server_settings.credentials ) if self.catalog.get_workspace('geonode') is None: self.catalog.create_workspace('geonode', 'http://www.geonode.org/') self.workspace = 'geonode' self.datastoreNames = [] def tearDown(self): """Clean up geoserver/geogig catalog. """ # delete stores (will cascade to delete layers) for store_name in self.datastoreNames: self.catalog.delete( self.catalog.get_store(store_name), recurse=True) # delete repository reference in geoserver for store_name in self.datastoreNames: self.remove_geogig_repo(store_name) # geoserver can leave connections open - HACK HACK HACK self.free_geogig_connections() # HACK HACK HACK -- sometimes connections from geoserver to geogig are left open. This kills the postgresql backend! # this is a major hammer. Once geoserver/geogig are better at cleaning up, remove this. def free_geogig_connections(self): with db.connections["geogig"].cursor() as c: c.execute( "select pg_terminate_backend(pid) from pg_stat_activity where application_name = 'PostgreSQL JDBC Driver' or application_name='geogig'") # aggressive delete of the repo (mostly cleans up after itself) # call the geogig rest API DELETE def remove_geogig_repo(self, ref_name): username = ogc_server_settings.credentials.username password = ogc_server_settings.credentials.password url = ogc_server_settings.rest http = httplib2.Http(disable_ssl_certificate_validation=False) http.add_credentials(username, password) netloc = urlparse(url).netloc http.authorizations.append( httplib2.BasicAuthentication( (username, password), netloc, url, {}, None, None, http )) rest_url = ogc_server_settings.LOCATION + \ "geogig/repos/" + ref_name + "/delete.json" resp, content = http.request(rest_url, 'GET') response = json.loads(content) token = response["response"]["token"] rest_url = ogc_server_settings.LOCATION + \ "geogig/repos/" + ref_name + "?token=" + token resp, content = http.request(rest_url, 'DELETE') # convenience method to load in the test dataset # return a (geonode) layer # the layer will be in Geoserver and Geonode # self.catalog.get_layer(layer.name) -- to get the Geoserver Layer def fully_import_file(self, path, fname, start_time_column, end_time_column=None): # setup time if end_time_column is None: time_config = {'convert_to_date': [start_time_column], 'start_date': start_time_column, 'configureTime': True} else: time_config = {'convert_to_date': [start_time_column, end_time_column], 'start_date': start_time_column, 'end_date': end_time_column, 'configureTime': True} name = os.path.splitext(fname)[0] + "_" + str(uuid.uuid1())[:8] self.datastoreNames.append(name) # remember for future deletion full_fname = os.path.join(path, fname) configs = self.prepare_file_for_import(full_fname) configs[0].update({'name': name}) configs[0].update({'layer_name': name}) configs[0].update(time_config) # configure the datastore/repo configs[0]['geoserver_store'] = {} configs[0]['geoserver_store']['type'] = 'geogig' configs[0]['geoserver_store']['name'] = name configs[0]['geoserver_store']['create'] = 'true' configs[0]['geoserver_store']['branch'] = 'master' configs[0]['geoserver_store']['geogig_repository'] = "geoserver://" + name result = self.generic_import(fname, path=path, configs=configs) return result def import_file(self, path, configs=None): """Imports the file. """ if configs is None: configs = [] self.assertTrue(os.path.exists(path), path) # run ogr2ogr ogr = OGRImport(path) layers = ogr.handle(configuration_options=configs) return layers def generic_import(self, filename, path, configs=None): if configs is None: configs = [{'index': 0}] path = os.path.join(path, filename) results = self.import_file(path, configs=configs) layer_results = [] for result in results: layer = Layer.objects.get(name=result[0]) self.assertEqual(layer.srid, 'EPSG:4326') self.assertTrue(layer.store in self.datastoreNames) self.assertEqual(layer.storeType, 'dataStore') if not path.endswith('zip'): self.assertGreaterEqual( layer.attributes.count(), DataSource(path)[0].num_fields ) layer_results.append(layer) return layer_results[0] def prepare_file_for_import(self, filepath): """ Prepares the file path provided for import; performs some housekeeping, uploads & configures the file. Returns a list of dicts of the form {'index': <layer_index>, 'upload_layer_id': <upload_layer_id>} these may be used as configuration options for importing all of the layers in the file. """ # Make a copy of the test file, as it's removed in configure_upload() filename = os.path.basename(filepath) tmppath = os.path.join('/tmp', filename) shutil.copy(filepath, tmppath) # upload & configure_upload expect closed file objects # This is heritage from originally being closely tied to a view passing request.Files of = open(tmppath, 'rb') of.close() files = [of] uploaded_data = self.upload(files, self.admin_user) self.configure_upload(uploaded_data, files) configs = [{'index': l.index, 'upload_layer_id': l.id} for l in uploaded_data.uploadlayer_set.all()] return configs def create_user(self, username, password, **kwargs): """Convenience method for creating users. """ user, created = User.objects.get_or_create(username=username, **kwargs) if created: user.set_password(password) user.save() return user
def _register_cascaded_service(url, type, name, username, password, wms=None, owner=None, parent=None): """ Register a service as cascading WMS """ try: service = Service.objects.get(base_url=url) return_dict = {} return_dict["service_id"] = service.pk return_dict["msg"] = "This is an existing Service" return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200) except: # TODO: Handle this error properly pass if wms is None: wms = WebMapService(url) # TODO: Make sure we are parsing all service level metadata # TODO: Handle for setting ServiceProfiletRole service = Service.objects.create( base_url=url, type=type, method="C", name=name, version=wms.identification.version, title=wms.identification.title, abstract=wms.identification.abstract, online_resource=wms.provider.url, owner=owner, parent=parent, ) service.keywords = ",".join(wms.identification.keywords) service.save() service.set_default_permissions() if type in ["WMS", "OWS"]: # Register the Service with GeoServer to be cascaded cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password) cascade_ws = cat.get_workspace(name) if cascade_ws is None: cascade_ws = cat.create_workspace(name, "http://geonode.org/" + name) # TODO: Make sure there isn't an existing store with that name, and # deal with it if there is try: cascade_store = cat.get_store(name, cascade_ws) except: cascade_store = cat.create_wmsstore(name, cascade_ws, username, password) cascade_store.capabilitiesURL = url cascade_store.type = "WMS" cat.save(cascade_store) available_resources = cascade_store.get_resources(available=True) elif type == "WFS": # Register the Service with GeoServer to be cascaded cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password) # Can we always assume that it is geonode? cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE) if cascade_ws is None: cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, "http://geonode.org/cascade") try: wfs_ds = cat.get_store(name, cascade_ws) except: wfs_ds = cat.create_datastore(name, cascade_ws) connection_params = { "WFSDataStoreFactory:MAXFEATURES": "0", "WFSDataStoreFactory:TRY_GZIP": "true", "WFSDataStoreFactory:PROTOCOL": "false", "WFSDataStoreFactory:LENIENT": "true", "WFSDataStoreFactory:TIMEOUT": "3000", "WFSDataStoreFactory:BUFFER_SIZE": "10", "WFSDataStoreFactory:ENCODING": "UTF-8", "WFSDataStoreFactory:WFS_STRATEGY": "nonstrict", "WFSDataStoreFactory:GET_CAPABILITIES_URL": url, } if username and password: connection_params["WFSDataStoreFactory:USERNAME"] = username connection_params["WFSDataStoreFactory:PASSWORD"] = password wfs_ds.connection_parameters = connection_params cat.save(wfs_ds) available_resources = wfs_ds.get_resources(available=True) # Save the Service record service, created = Service.objects.get_or_create(type=type, method="C", base_url=url, name=name, owner=owner) service.save() service.set_default_permissions() elif type == "WCS": return HttpResponse("Not Implemented (Yet)", status=501) else: return HttpResponse( "Invalid Method / Type combo: " + "Only Cascaded WMS, WFS and WCS supported", mimetype="text/plain", status=400, ) message = "Service %s registered" % service.name return_dict = [ { "status": "ok", "msg": message, "service_id": service.pk, "service_name": service.name, "service_title": service.title, "available_layers": available_resources, } ] if settings.USE_QUEUE: # Create a layer import job WebServiceHarvestLayersJob.objects.get_or_create(service=service) else: _register_cascaded_layers(service) return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200)
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update( host="localhost", port="5432", database="db", user="******", passwd="password", dbtype="postgis") self.cat.save(ds) def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update( host="localhost", port="5432", database="db", user="******", passwd="password", dbtype="postgis") self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store(ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) def testCoverageStoreCreate(self): ds = self.cat.create_coveragestore2("coverage_gsconfig") ds.data_url = "file:data/mytiff.tiff" self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf) ) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf) ) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") # TODO: Uploading WorldImage file no longer works??? # ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) # self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) # self.assertRaises( # ConflictingDataError, # lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) # ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf) ) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff) ) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = "Not the original attribution" # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles)
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None, skip_unadvertised=False): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ # avoid circular import problem from geonode.layers.models import set_attributes if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." cat = Catalog(ogc_server_settings.internal_rest, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if filter: resources = [k for k in resources if filter in k.name] # filter out layers depending on enabled, advertised status: resources = [k for k in resources if k.enabled == "true"] if skip_unadvertised: resources = [k for k in resources if k.advertised == "true" or k.advertised == None] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = { 'stats': { 'failed':0, 'updated':0, 'created':0, }, 'layers': [] } start = datetime.datetime.now() for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create(name=name, defaults = { "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() # recalculate the layer statistics set_attributes(layer, overwrite=True) except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception('Failed to process %s' % resource.name.encode('utf-8'), e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' output['stats']['created']+=1 else: status = 'updated' output['stats']['updated']+=1 msg = "[%s] Layer %s (%d/%d)" % (status, name, i+1, number) info = {'name': name, 'status': status} if status == 'failed': output['stats']['failed']+=1 info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output['layers'].append(info) if verbosity > 0: print >> console, msg
def get_selected_raster(region, variable, date): try: conn = psycopg2.connect( "dbname={0} user={1} host={2} password={3}".format( cfg.connection['dbname'], cfg.connection['user'], cfg.connection['host'], cfg.connection['password'])) cur = conn.cursor() storename = region + '_' + variable + '_' + date cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password']) try: something = cat.get_store(storename, cfg.geoserver['workspace']) if not something: print "No store" else: mean, stddev, min, max = get_vic_summary( region, variable, date) return storename, mean, stddev, min, max except geoserver.catalog.FailedRequestError as e: try: sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format( region, variable, date) cur.execute(sql) data = cur.fetchall() mean, stddev, min, max = get_vic_summary( region, variable, date) rest_url = cfg.geoserver['rest_url'] if rest_url[-1] != "/": rest_url = rest_url + '/' headers = { 'Content-type': 'image/tiff', } request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format( rest_url, cfg.geoserver['workspace'], storename) # Creating the rest url user = cfg.geoserver['user'] password = cfg.geoserver['password'] requests.put( request_url, headers=headers, data=data[0][0], auth=(user, password)) # Creating the resource on the geoserver conn.close() return storename, mean, stddev, min, max except Exception as e: print e return e # if storename in geoserver_engine.list_stores()['result']: # mean, stddev, min, max = get_vic_summary(region,variable,date) # return storename,mean, stddev, min, max # else: # sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format(region,variable,date) # cur.execute(sql) # data = cur.fetchall() # # mean, stddev, min, max = get_vic_summary(region,variable,date) # # rest_url = cfg.geoserver['rest_url'] # # if rest_url[-1] != "/": # rest_url = rest_url + '/' # # # headers = { # 'Content-type': 'image/tiff', # } # request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format(rest_url,cfg.geoserver['workspace'] ,storename) # Creating the rest url # # user = cfg.geoserver['user'] # password = cfg.geoserver['password'] # requests.put(request_url, headers=headers, data=data[0][0], # auth=(user,password)) # Creating the resource on the geoserver # # conn.close() # return storename,mean,stddev,min,max except Exception as e: print e return e
class GsConn: def __init__(self, host, login, password, debug=False): """ Geoserver connection """ self.host = host self.login = login self.password = password self.debug = debug # Connect to server self.cat = Catalog("http://%s/geoserver/rest" % host, login, password) if self.debug is True: print "Connected to geoserver" def crate_workspace(self, name, overwrite=False): """ Creates a workspace :param name: Workspace name. :param overwrite: If True, delete existing workspace. :return: None """ workspaces = [workspace.name for workspace in self.cat.get_workspaces()] if name in workspaces and overwrite is True: # ws2del = self.cat.get_workspace(name) # self.cat.delete(ws2del, purge=True, recurse=True) return None # NOTE: If we delete the workspace then all associated layers are lost. elif name in workspaces and overwrite is False: print "ERROR: Workspace %s already exists (use overwrite=True)." % name self.cat.create_workspace(name, "http://%s/%s" % (self.host, name)) if self.debug is True: print "Workspace %s available." % name ws = self.cat.get_workspace(name) ws.enabled = True def create_pg_store(self, name, workspace, host, port, login, password, dbname, schema, overwrite=False): """ Creates datastore. :param name: Name of the datastore. :param workspace: Name of the workspace to use. :param overwrite: If True replace datastore. :return: None """ stores = [store.name for store in self.cat.get_stores()] if name in stores and overwrite is True: # st2del = self.cat.get_store(name) # self.cat.delete(st2del, purge=True, recurse=True) # self.cat.reload() return None # NOTE: If we delete store, every layers associated with are lost. elif name in stores and overwrite is False: print "ERROR: Store %s already exists (use overwrite=True)." % name ds = self.cat.create_datastore(name, workspace) ds.connection_parameters.update( host=host, port=port, user=login, passwd=password, dbtype="postgis", database=dbname, schema=schema ) self.cat.save(ds) ds = self.cat.get_store(name) if ds.enabled is False: print "ERROR: Geoserver store %s not enabled" % name if self.debug is True: print "Datastore %s created." % name def publish_pg_layer(self, layer_table, layer_name, store, srid, overwrite=True): """ """ existing_lyr = self.cat.get_layer("ma_carte:%s" % layer_table) if existing_lyr is not None: print "Layer ma_carte:%s already exists, deleting it." % layer_table self.cat.delete(existing_lyr) self.cat.reload() ds = self.cat.get_store(store) ft = self.cat.publish_featuretype(layer_table, ds, "EPSG:%s" % srid, srs="EPSG:4326") ft.projection_policy = "REPROJECT_TO_DECLARED" ft.title = layer_name self.cat.save(ft) if ft.enabled is False: print "ERROR: Layer %s %s %s is not enabled."(ft.workspace.name, ft.store.name, ft.title) if self.debug is True: print "Layer %s>%s>%s published." % (ft.workspace.name, ft.store.name, ft.title) def create_style_from_sld(self, style_name, sld_file, workspace, overwrite=True): """ """ if self.cat.get_style(style_name) is not None: print "Style %s already exists, deleting it." % style_name style2del = self.cat.get_style(style_name) self.cat.delete(style2del) self.cat.create_style( style_name, open(sld_file).read(), overwrite=overwrite ) # FIXME: if ", workspace=workspace" specified can't delete style if self.debug is True: print "Style %s created in Geoserver" % style_name def apply_style_to_layer(self, layer_name, style_name): """ Apply a geoserver styler to a layer """ gs_layer = self.cat.get_layer(layer_name) gs_style = self.cat.get_style(style_name) # FIXME: Which works better? # gs_layer.default_style = gs_style / gs_layer._set_default_style(gs_style) # FIXME: Maybe indicate workspace when saving style then name the style as "workspace:style" gs_layer._set_default_style(gs_style) self.cat.save(gs_layer) if self.debug is True: print "Style applied to %s" % layer_name
cat.save(that_layer) ooitest = cat.get_workspace("ooi_test") data = cat.get_resources(store="asd",workspace="geonode") print data layer = cat.get_layer("1k time varying test") print layer.href cat.add_data_to_store print layer #drop those things not of interest wmstest = cat.get_workspace("geonode") try: wmsstore = cat.get_store("ooi") cat.delete(wmsstore) except: print "opps" #create the data store wmsstore = cat.create_datastore("ooi", wmstest) wmsstore.capabilitiesURL = "http://www.geonode.org/" wmsstore.type = "PostGIS" #connection info params = { 'Connection timeout': '20', 'Estimated extends': 'true', 'Expose primary keys': 'false',
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." url = "%srest" % settings.OGC_SERVER['default']['LOCATION'] cat = Catalog(url, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if filter: resources = [k for k in resources if filter in k.name] # filter out layers explicitly disabled by geoserver resources = [k for k in resources if k.enabled == "true"] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = [] for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create( name=name, defaults={ "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name.encode('utf-8'), resource.name.encode('utf-8')), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception( 'Failed to process %s' % resource.name.encode('utf-8'), e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' else: status = 'updated' msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number) info = {'name': name, 'status': status} if status == 'failed': info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output.append(info) if verbosity > 0: print >> console, msg
class GsConfigTestCase(unittest.TestCase): """Test of the file manager""" gs = None workdir = None def setUp(self): cfg = ConfigParser.SafeConfigParser() cfg.read((os.path.join(TEST_DIR,"tests.cfg"))) self.gs = GsConfig(cfg) self.workdir = os.path.abspath(os.path.join(TEST_DIR,"workdir","data")) gsUrl = self.gs.config.get("GeoServer", "url") gsUser = self.gs.config.get("GeoServer", "user") gsPassword = self.gs.config.get("GeoServer", "password") self.direct_gs = Catalog(gsUrl, gsUser, gsPassword) def tearDown(self): class Req: href=self.gs.config.get("GeoServer", "url") + "/styles/tmp_polygon" if self.direct_gs.get_style("tmp_polygon"): self.direct_gs.delete(Req()) ### LAYERS ### def test_getLayers(self): layers = self.gs.getLayers() print layers # TODO: Add test def test_getLayer(self): layers = self.gs.getLayers() layer = self.gs.getLayer(layers[0].name) print layer # TODO: Add test ### FEATURE STORE ### def test_createFeatrueStore(self): # TODO: make sure that source files exist # check / delete layer line_layer = self.direct_gs.get_layer("line_crs") if line_layer is not None: self.direct_gs.delete(line_layer) # check / delete feature type # # sorry - it is not possible to delete feature type with gsconfig.py # # example in test # https://github.com/dwins/gsconfig.py/blob/master/test/catalogtests.py#L424 # only shows how to disable it # # FOR NOW, PLEASE REMOVE THE FEATURE TYPE "line_crs0" MANUALLY # #line_feature = self.direct_gs.get_featuretype("line_crs0") # this function does not exist #if line_feature is not None: # self.direct_gs.delete(line_feature) # TODO: remove the feature type using the gs rest api itself # check / delete data store # line_store = self.direct_gs.get_store("line_crs") # if line_store is not None: # self.direct_gs.delete(line_store) self.assertEquals( True, None == self.direct_gs.get_layer("line_crs"), "The layer line_crs already exists. Please, remove it manually." ) self.assertEquals( True, None == self.direct_gs.get_store("line_crs"), "The store line_crs already exists. Please, remove it manually." ) self.gs.createFeatureStore(self.workdir+"/line_crs","TestWS","line_crs") # Another problem - gs.createFeatureStore DOES NOT return the resource URI # - hence WE DON'T know the name of the created layer! # TODO: use the gs rest api itself and find out the name of the created layer # - whether it is line_crs0, line_crs1 or what... self.assertEquals( False, None == self.direct_gs.get_layer("line_crs"), "The layer line_crs is not there. Was it created under another name?" ) ### STYLES ### def test_getStyle(self): """Test get style""" style = self.gs.getStyle("line") self.assertTrue(style.find("<sld:Name>line</sld:Name>")>0) def test_postStyle(self): """Test get files function""" global style self.gs.postStyle("tmp_polygon",style) # TODO: add test # FIXME: UploadError: SLD file styles/tmp_polygon.sld.sld already exists. def test_putStyle(self): """Test get files function""" global style self.gs.postStyle("tmp_polygon",style) self.gs.putStyle("tmp_polygon",style)
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testAbout(self): about_html = self.cat.about() self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") self.cat.get_store(workspace="best workspace ever", name="best store ever") self.assertRaises(FailedRequestError, lambda: self.cat.get_resource( workspace="best workspace ever", store="best store ever", name="best resource ever")) self.cat.get_layer("best layer ever") self.cat.get_layergroup("best layergroup ever") def testUnicodeUrl(self): """ Tests that the geoserver.support.url function support unicode strings. """ # Test the url function with unicode seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml'] u = url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml") # Test the url function with normal string seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'] u = url(base=self.cat.service_url, seg=seg) self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name)
class GeoServer(Server): """ Represents a server running GeoServer [1], an application that provides access to layers. This class provides a concrete implementation of the more generic Server component (which is intentionally generic). Currently the Server class does not dictate an interface for accessing resources but this class aims to present GeoServer specific components (such as workspaces) as generic components (such as namespaces). GeoServer instances typically represent individual instances (i.e. hosts are servers) rather than a wider and more abstract platform offered by a service provider. Information on layers and other resources are fetched using a combination of the GeoServer specific administrative API [2] accessed through geoserver-restconfig [3] and OGC services accessed through OWSLib [4] (and currently limited to WMS and WFS). [1] https://geoserver.readthedocs.io/en/latest/ [2] https://geoserver.readthedocs.io/en/latest/rest/index.html [3] https://pypi.org/project/geoserver-restconfig [4] https://pypi.org/project/OWSLib/ """ def __init__( self, server_id: str, label: str, hostname: str, port: str, api_path: str, wms_path: str, wfs_path: str, username: str, password: str, ): """ :param server_id: unique identifier, typically a ULID (Universally Unique Lexicographically Sortable Identifier) :param label: a human readable, well-known, identifier for the server - typically based on the hostname :param hostname: servers fully qualified hostname :param port: port on which GeoServer is running (usually '80' or '8080') :param api_path: URL path, relative to the root of the server, to the GeoServer API (usually '/geoserver/rest') :param wms_path: URL path, relative to the root of the server, to the GeoServer WMS endpoint (usually '/geoserver/ows?service=wms&version=1.3.0&request=GetCapabilities') :param wfs_path: URL path, relative to the root of the server, to the GeoServer WFS endpoint (usually '/geoserver/ows?service=wfs&version=2.0.0&request=GetCapabilities') :param username: username for account to use for GeoServer API :param password: password for account to use for GeoServer API """ endpoint = build_base_data_source_endpoint(data_source={ "hostname": hostname, "port": port }) self.client = Catalogue(service_url=f"{endpoint}{api_path}", username=username, password=password) self.wms = WebMapService(url=f"{endpoint}{wms_path}", version="1.3.0", username=username, password=password) self.wfs = WebFeatureService(url=f"{endpoint}{wfs_path}", version="2.0.0", username=username, password=password) super().__init__( server_id=server_id, label=label, hostname=hostname, server_type=ServerType.GEOSERVER.value, version=self._get_geoserver_version(), ) def get_namespaces(self) -> List[str]: """ Gets all GeoServer workspace names as Namespace labels :return: list of Namespace labels """ workspaces = [] for workspace in self.client.get_workspaces(): workspaces.append(workspace.name) return workspaces def get_namespace(self, namespace_reference: str) -> Dict[str, str]: """ Gets a specific workspace as a Namespace Note: GeoServer workspaces do not support the concept of a title, a static substitute value is therefore used Note: GeoServer workspaces do support the concept of a namespace, but it is not yet implemented [#28] :param namespace_reference: Namespace (workspace) label (name) :return: dictionary of Namespace information that can be made into a Namespace object """ workspace = self.client.get_workspace(name=namespace_reference) if workspace is None: raise KeyError( f"Namespace [{namespace_reference}] not found in server [{self.label}]" ) return {"label": workspace.name, "title": "-", "namespace": "-"} def get_repositories(self) -> List[Tuple[str, str]]: """ Gets all GeoServer store names as Repository labels :return: list of Repository:Namespace label tuples """ stores = [] # Passing workspaces here is a workaround for a bug in the get stores method where workspaces aren't specified. # The method says all workspaces should be checked but the logic to do this is in the wrong place so none are. for store in self.client.get_stores( workspaces=self.client.get_workspaces()): stores.append((store.name, store.workspace.name)) return stores def get_repository(self, repository_reference: str, namespace_reference: str) -> Dict[str, str]: """ Gets a specific store as a Repository If a Namespace (workspace) label is specified the Repository must exist within that Namespace. GeoServer store types are sometimes unsuitable or non-standard and so need to be mapped to a conventional value. in the RepositoryType enum using the GeoServerRepositoryType enum. Note: GeoServer stores do not support the concept of a title, a static substitute value is therefore used Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33] :param repository_reference: Repository (store) label (name) :param namespace_reference: Namespace (store) label (name) :return: dictionary of repository information that can be made into a Repository object """ _store = self.client.get_store(name=repository_reference, workspace=namespace_reference) if _store is None: raise KeyError( f"Repository [{repository_reference}] not found in server [{self.label}]" ) store = { "label": _store.name, "title": "-", "repository_type": RepositoryType[GeoServerRepositoryType(str( _store.type).lower()).name].value, "namespace_label": _store.workspace.name, } if hasattr(_store, "description") and _store.description is not None: store["title"] = _store.description if (store["repository_type"] == RepositoryType.POSTGIS.value or store["repository_type"] == RepositoryType.ORACLE.value): store["hostname"] = _store.connection_parameters["host"] store["database"] = _store.connection_parameters["database"] store["schema"] = _store.connection_parameters["schema"] return store def get_styles(self) -> List[Tuple[str, Optional[str]]]: """ Gets all GeoServer style names as Style labels Python's None value will be used to represent the Namespace of global styles (i.e that don't have a Namespace (workspace)). :return: list of Style:Namespace label tuples """ styles = [] for _style in self.client.get_styles(): styles.append((_style.name, _style.workspace)) return styles def get_style(self, style_reference: str, namespace_reference: str = None) -> Dict[str, str]: """ Gets a specific style as a Style If a Namespace (workspace) label is specified the Style must exist within that Namespace. Note: GeoServer styles do support the concept of a title, but it is not exposed through the admin API so a static substitute value is therefore used Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33] :param style_reference: Style (style) label (name) :param namespace_reference: Namespace (store) label (name) :return: dictionary of style information that can be made into a Style object """ _style = self.client.get_style(name=style_reference, workspace=namespace_reference) _type = str(_style.style_format).lower() if _type == "sld10": _type = "sld" style = { "label": _style.name, "title": "-", "style_type": _type, } if hasattr(_style, "workspace") and _style.workspace is not None: style["namespace_label"] = _style.workspace return style def get_layers(self) -> List[str]: """ Gets all GeoServer layer names as Layer labels :return: list of Layer labels """ layers = [] for _layer in self.client.get_layers(): layers.append(_layer.name) return layers def get_layer( self, layer_reference: str ) -> Dict[str, Union[Optional[str], List[str], List[Tuple[ str, Optional[str]]]]]: """ Gets a specific layer as a Layer Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33] :param layer_reference: Layer (layer) label (name) :return: dictionary of layer information that can be made into a Layer object """ _layer = self.client.get_layer(name=layer_reference) layer = { "label": _layer.resource.name, "title": _layer.resource.title, "layer_type": str(_layer.type).lower(), "geometry_type": None, "services": [], "table_view": None, "namespace_label": _layer.resource.workspace.name, "repository_label": _layer.resource.store.name, "style_labels": [(_layer.default_style.name, _layer.default_style.workspace)], } if layer_reference in list( self.wms.contents ) or f"{_layer.resource.workspace.name}:{layer_reference}" in list( self.wms.contents): layer["services"].append(LayerService.WMS.value) if layer_reference in list( self.wfs.contents ) or f"{_layer.resource.workspace.name}:{layer_reference}" in list( self.wfs.contents): layer["services"].append(LayerService.WFS.value) # WFS lookups don't seem to mind if the layer is namespaced or not _properties = self.wfs.get_schema(layer_reference) if "geometry" in _properties and isinstance( _properties["geometry"], str): try: layer["geometry_type"] = LayerGeometry[ GeoServerLayerGeometry(str( _properties["geometry"])).name].value except ValueError: raise ValueError( f"Geometry [{_properties['geometry']}] for layer {layer_reference} not mapped to " f"LayerGeometry enum.") elif "properties" in _properties: for geometry_column_name in GeoServerGeometryColumnNames: if geometry_column_name.value in _properties[ "properties"].keys(): try: layer["geometry_type"] = LayerGeometry[ GeoPropertyGeoServerLayerGeom( str(_properties["properties"][ geometry_column_name.value]) ).name].value except ValueError: raise ValueError( f"Geometry [{_properties['properties'][geometry_column_name.value]}] for layer " f"{layer_reference} in column '{geometry_column_name.value}' not mapped to " f"LayerGeometry enum.") if (str(_layer.resource.store.type).lower() == RepositoryType.POSTGIS.value or str(_layer.resource.store.type).lower() == RepositoryType.ORACLE.value): layer["table_view"] = _layer.resource.native_name return layer def get_layer_groups(self) -> List[Tuple[str, Optional[str]]]: """ Gets all GeoServer layer group names as LayerGroup labels Python's None value will be used to represent the Namespace of global layer groups (i.e that don't have a Namespace (workspace)). :return: list of LayerGroup:Namespace label tuples """ layer_groups = [] for _layer_group in self.client.get_layergroups( workspaces=self.client.get_workspaces()): layer_groups.append((_layer_group.name, _layer_group.workspace)) return layer_groups def get_layer_group( self, layer_group_reference: str, namespace_reference: str ) -> Dict[str, Union[Optional[str], List[str], List[Tuple[ str, Optional[str]]]]]: """ Gets a specific layer group as a LayerGroup If a Namespace (workspace) label is specified the LayerGroup must exist within that Namespace. Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33] :param layer_group_reference: LayerGroup (layer group) label (name) :param namespace_reference: Namespace (store) label (name) :return: dictionary of layer group information that can be made into a LayerGroup object """ _layer_group = self.client.get_layergroup( name=layer_group_reference, workspace=namespace_reference) layer_group = { "label": _layer_group.name, "title": _layer_group.title, "services": [], "namespace_label": _layer_group.workspace, "layer_labels": [], "style_labels": [], } for layer_label in _layer_group.layers: layer_label = layer_label.split(":") if len(layer_label) == 2: layer_group["layer_labels"].append( (layer_label[1], layer_label[0])) elif len(layer_label) == 1: layer_group["layer_labels"].append((layer_label[0], None)) if f"{namespace_reference}:{layer_group_reference}" in list( self.wms.contents): layer_group["services"].append(LayerService.WMS.value) if f"{namespace_reference}:{layer_group_reference}" in list( self.wfs.contents): layer_group["services"].append(LayerService.WFS.value) _properties = self.wfs.get_schema( f"{namespace_reference}:{layer_group_reference}") try: layer_group["geometry_type"] = LayerGeometry[ GeoServerLayerGeometry(str( _properties["geometry"])).name].value except ValueError: raise ValueError( f"Geometry [{_properties['geometry']}] not mapped to LayerGeometry enum." ) for style_label in _layer_group.styles: if style_label is not None: style_label = style_label.split(":") if len(style_label) == 2 and ( style_label[1], style_label[0]) not in layer_group["style_labels"]: layer_group["style_labels"].append( (style_label[1], style_label[0])) if len(style_label) == 1 and ( style_label[0], None) not in layer_group["style_labels"]: layer_group["style_labels"].append((style_label[0], None)) return layer_group def _get_geoserver_version(self) -> str: """ Gets the GeoServer version :return: GeoServer version string """ return self.client.get_version()
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) self.cat.delete(ds) def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store( ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) def testCoverageStoreCreate(self): ds = self.cat.create_coveragestore2("coverage_gsconfig") ds.data_url = "file:data/mytiff.tiff" self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set( ['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_( self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore( "states_test", shapefile_plus_sidecars, sf)) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore( "states_raster_test", shapefile_plus_sidecars, sf)) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") # TODO: Uploading WorldImage file no longer works??? # ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) # self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) # self.assertRaises( # ConflictingDataError, # lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) # ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore( "Pk50095_vector", tiffdata, sf)) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = "Not the original attribution" # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assert_(f is not None) self.assertEqual(f.name, fred.name) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assert_(jed is None) jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assert_(self.cat.get_style("jed", workspace="topp") is None) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assert_(f is not None) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assert_(ned is not None) self.assert_(zed is not None) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testTimeDimension(self): sf = self.cat.get_workspace("sf") files = shapefile_and_friends( os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer( 'boxes_with_end_date').resource # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute) def testImageMosaic(self): # testing the mosaic creation name = 'cea_mosaic' data = open('test/data/mosaic/cea.zip', 'rb') self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assert_(resource is not None) # delete granule from mosaic coverage = name store = name granule_id = name + '.1' self.cat.mosaic_delete_granule(coverage, store, granule_id)
class GeoserverHelper: def __init__(self, geoserverUrl = "", geoserverUserName = "", geoserverPW = "", geoserverWorkSpace = "", postgreIP = "", postgreUserName = "", postgrePW = ""): """use the constructor given arguments if used""" self.geoserverUrl = geoserverUrl if geoserverUrl != "" else GEOSERVERURL self.geoserverUserName = geoserverUserName if geoserverUserName != "" else GEOSERVERUSERNAME self.geoserverPW = geoserverPW if geoserverPW != "" else GEOSERVERPW self.geoserverWorkSpace = geoserverWorkSpace if geoserverWorkSpace != "" else "crc" self.postgreIP = postgreIP if postgreIP != "" else POSTGREIP self.postgreUserName = postgreUserName if postgreUserName != "" else POSTGREUSERNAME self.postgrePW = postgrePW if postgrePW != "" else POSTGREPW if self.geoserverUrl[-1] != '/': raise Exception("GeoserverUrl must end with a slash ('/')") self.catalog = Catalog(self.geoserverUrl+"rest/") self.catalog.http.add_credentials(self.geoserverUserName,self.geoserverPW) try: workspaces = self.catalog.get_workspaces() except: e = sys.exc_info()[0] print e self.cWorkSpace = self.catalog.get_workspace(self.geoserverWorkSpace) def getLayers(self): return self.cWorkSpace.catalog.get_layers() def insertShapeIntoPostGis(self, shapeFile, databaseName, tableName, encoding=3857): '''returns the returnCode of the execution of the insert script, e.g: helper.insertShapeIntoPostGis('/home/c815/gsTest/test.shp','crc','testingHelper2')''' if not os.path.isfile(shapeFile): print "Shape file not found" return -1 cmds = "PGPASSWORD={pgPW} ./createWSFTFromSHP.sh -s {shapeFileF} -d {databaseNameF} -t {tableNameF} -u {postgreUsername} -i {postgreIP}".format(pgPW=self.postgrePW, shapeFileF=shapeFile,databaseNameF=databaseName, tableNameF=tableName, postgreUsername=self.postgreUserName, postgreIP=self.postgreIP) return subprocess.call(cmds, shell=True) def uploadShapeFile(self, shapeFile, storeName): shpPlusSidcars = geoserver.util.shapefile_and_friends(shapeFile[:-3]) shpPlusSidcars self.ft = self.catalog.create_featurestore(storeName, shpPlusSidcars, self.cWorkSpace) def getStyles(self): return self.catalog.get_styles() def uploadStyleFile(self, sldFile, styleName, overWrite, workSpace = None): f = open(sldFile,'r') styleSrc = f.read() uploadStyle(styleSrc, styleName, overWrite, workSpace) f.close() def uploadStyle(self, sldSrc, styleName, overWrite, workSpace = None): self.catalog.create_style(styleName,sldSrc, overWrite, workSpace) def publishPostGISLayer(self,postGISLayerName, storeName, crs='EPSG:3857'): '''cat.publish_featuretype('testingstuff',crcStore,native_crs='EPSG:3857')''' store = self.catalog.get_store(storeName) if store != None: self.catalog.publish_featuretype(postGISLayerName, store, crs) def setDefaultStyleForLayer(self, layerName, styleName): l = self.catalog.get_layer(layerName) sNames = [ i.name for i in self.getStyles() ] if styleName not in sNames: split = styleName.split(':') if len(split) == 2: workSpace = styleName.split(':')[0] newStyleName = styleName.split(':')[1] else: return -1 style = self.catalog.get_style(newStyleName, workSpace) if style is None: return -1 if l != None: l._set_default_style(styleName) self.catalog.save(l) return 0 def createPostGISDataStore(self, storeName, postGisPassword, postGisUser, postGisHost, postGisDatabase, workSpace = None): #check if connection parameter are valid try: conn = psycopg2.connect("dbname='{dbName}' user='******' host='{Host}' password='******'".format(dbName=postGisDatabase, dbUser=postGisUser, Host=postGisHost, password=postGisPassword)) except: return False w = self.catalog.create_datastore(storeName, workSpace) template = Template("""{'validate connections': 'true', 'port': '5432', 'Support on the fly geometry simplification': 'true', 'create database': 'false', 'dbtype': 'postgis', 'Connection timeout': '20', 'namespace': 'http://www.crcproject.com', 'Max connection idle time': '300', 'Expose primary keys': 'false', 'min connections': '1', 'Max open prepared statements':'50', 'passwd': '$passwd', 'encode functions': 'false', 'max connections': '10', 'Evictor tests per run': '3', 'Loose bbox': 'true', 'Evictor run periodicity': '300', 'Estimated extends': 'true', 'database': '$database', 'fetch size': '1000', 'Test while idle': 'true', 'host': '$host', 'preparedStatements': 'false', 'schema': 'public', 'user': '******'}""") dic = ast.literal_eval(template.substitute( passwd=postGisPassword, user=postGisUser, host=postGisHost, database=postGisDatabase)) #'passwd': 'crypt1:Bsaz2AUI8T+6Pj43krA7kg==', #'user': '******'} #'database': 'crc', #'host': 'localhost', w.connection_parameters = dic self.catalog.save(w) return True
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None): """Configure the layers available in GeoServer in GeoNode. It returns a list of dictionaries with the name of the layer, the result of the operation and the errors and traceback if it failed. """ if console is None: console = open(os.devnull, 'w') if verbosity > 1: print >> console, "Inspecting the available layers in GeoServer ..." url = "%srest" % settings.GEOSERVER_BASE_URL cat = Catalog(url, _user, _password) if workspace is not None: workspace = cat.get_workspace(workspace) resources = cat.get_resources(workspace=workspace) elif store is not None: store = cat.get_store(store) resources = cat.get_resources(store=store) else: resources = cat.get_resources(workspace=workspace) if filter: resources = [k for k in resources if filter in k.name] # filter out layers explicitly disabled by geoserver resources = [k for k in resources if k.enabled == "true"] # TODO: Should we do something with these? # i.e. look for matching layers in GeoNode and also disable? disabled_resources = [k for k in resources if k.enabled == "false"] number = len(resources) if verbosity > 1: msg = "Found %d layers, starting processing" % number print >> console, msg output = [] for i, resource in enumerate(resources): name = resource.name store = resource.store workspace = store.workspace try: # Avoid circular import problem from geonode.layers.models import Layer layer, created = Layer.objects.get_or_create(name=name, defaults = { "workspace": workspace.name, "store": store.name, "storeType": store.resource_type, "typename": "%s:%s" % (workspace.name, resource.name), "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": owner, "uuid": str(uuid.uuid4()) }) layer.save() except Exception, e: if ignore_errors: status = 'failed' exception_type, error, traceback = sys.exc_info() else: if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print >> sys.stderr, msg raise Exception('Failed to process %s' % resource.name, e), None, sys.exc_info()[2] else: if created: layer.set_default_permissions() status = 'created' else: status = 'updated' msg = "[%s] Layer %s (%d/%d)" % (status, name, i+1, number) info = {'name': name, 'status': status} if status == 'failed': info['traceback'] = traceback info['exception_type'] = exception_type info['error'] = error output.append(info) if verbosity > 0: print >> console, msg
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assertIsNone(self.cat.get_layer('import')) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assertIsNotNone(self.cat.get_layer('import')) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. try: if lyr: self.cat.delete(lyr) self.cat.delete(lyr.resource) if ds: self.cat.delete(ds) except Exception: pass def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store( ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) @drop_table('import2') def testVirtualTables(self): ds = self.cat.create_datastore("gsconfig_import_test2") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test2") self.cat.add_data_to_store( ds, "import2", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) store = self.cat.get_store("gsconfig_import_test2") geom = JDBCVirtualTableGeometry('the_geom', 'MultiPolygon', '4326') ft_name = 'my_jdbc_vt_test' epsg_code = 'EPSG:4326' sql = "select * from import2 where 'STATE_NAME' = 'Illinois'" keyColumn = None parameters = None jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters) self.cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt) # DISABLED; this test works only in the very particular case # "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR # def testCoverageStoreCreate(self): # ds = self.cat.create_coveragestore2("coverage_gsconfig") # ds.data_url = "file:test/data/mytiff.tiff" # self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual([ ("text/xml", "TC211", "http://example.com/gsconfig.test.metadata") ], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set( ['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEqual(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEqual(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEqual(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501 ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501 wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(workspace=wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: # sanitize the layer name - validation will fail on newer geoservers name = layer.replace(':', '_') self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) changed_layer = added_layers[0] self.assertEqual(True, changed_layer.advertised) self.assertEqual(True, changed_layer.enabled) changed_layer.advertised = False changed_layer.enabled = False self.cat.save(changed_layer) self.cat._cache.clear() changed_layer = wmsstore.get_resources()[0] changed_layer.fetch() self.assertEqual(False, changed_layer.advertised) self.assertEqual(False, changed_layer.enabled) # Testing projection and projection policy changes changed_layer.projection = "EPSG:900913" changed_layer.projection_policy = "REPROJECT_TO_DECLARED" self.cat.save(changed_layer) self.cat._cache.clear() layer = self.cat.get_layer(changed_layer.name) self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy) self.assertEqual(layer.resource.projection, changed_layer.projection) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.items(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assertIsNotNone(self.cat.get_resource("states_test", workspace=sf)) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore( "states_test", shapefile_plus_sidecars, sf)) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore( "states_raster_test", shapefile_plus_sidecars, sf)) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assertIsNone(self.cat.get_layer("states_test")) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") self.cat.create_coveragestore("Pk50095", tiffdata, sf) self.assertIsNotNone(self.cat.get_resource("Pk50095", workspace=sf)) self.assertRaises( ConflictingDataError, lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)) self.assertRaises( UploadError, lambda: self.cat.create_featurestore( "Pk50095_vector", tiffdata, sf)) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)) self.cat.create_coveragestore_external_geotiff( "Pk50095_ext", 'file:test/data/Pk50095.tif', sf) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = { 'title': 'Not the original attribution', 'width': '123', 'height': '321', 'href': 'http://www.georchestra.org', 'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg', 'type': 'image/jpeg' } # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = next( (s for s in lyr.styles if s.name == "pophatch")) lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # check count before tests (upload) count = len(self.cat.get_styles()) # upload new style, verify existence with open("test/fred.sld") as fred_sld: self.cat.create_style("fred", fred_sld.read()) fred = self.cat.get_style("fred") self.assertIsNotNone(fred) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes with open("test/ted.sld") as ted_sld: self.cat.create_style("fred", ted_sld.read(), overwrite=True) fred = self.cat.get_style("fred") self.assertIsNotNone(fred) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assertIsNone(self.cat.get_style("fred")) # attempt creating new style with open("test/fred.sld") as fred_sld: self.cat.create_style("fred", fred_sld.read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assertIsNotNone(f) self.assertEqual(f.name, fred.name) # compare count after upload self.assertEqual(count + 1, len(self.cat.get_styles())) # attempt creating a new style without "title" with open("test/notitle.sld") as notitle_sld: self.cat.create_style("notitle", notitle_sld.read()) notitle = self.cat.get_style("notitle") self.assertEqual(None, notitle.sld_title) def testWorkspaceStyles(self): # upload new style, verify existence with open("test/fred.sld") as fred_sld: self.cat.create_style("jed", fred_sld.read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assertIsNone(jed) jed = self.cat.get_style("jed", workspace="topp") self.assertIsNotNone(jed) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assertIsNotNone(jed) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes with open("test/ted.sld") as ted_sld: self.cat.create_style("jed", ted_sld.read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertIsNotNone(jed) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assertIsNone(self.cat.get_style("jed", workspace="topp")) # attempt creating new style with open("test/fred.sld") as fred_sld: self.cat.create_style("jed", fred_sld.read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assertIsNotNone(f) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence with open("test/fred.sld") as fred_sld: self.cat.create_style("ned", fred_sld.read(), overwrite=True, workspace="topp") with open("test/fred.sld") as ted_sld: self.cat.create_style("zed", ted_sld.read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assertIsNotNone(ned) self.assertIsNotNone(zed) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() if lyr.default_style is not None: self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assertIsNone(ws) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assertTrue(states.enabled) states.enabled = False self.assertFalse(states.enabled) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assertFalse(states.enabled) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assertTrue(states.enabled) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads' ], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) def testImageMosaic(self): """ Test case for Issue #110 """ # testing the mosaic creation name = 'cea_mosaic' with open('test/data/mosaic/cea.zip', 'rb') as data: self.cat.create_imagemosaic(name, data) # get the layer resource back self.cat._cache.clear() resource = self.cat.get_layer(name).resource self.assertIsNotNone(resource) # delete granule from mosaic coverage = name store = self.cat.get_store(name) granules = self.cat.list_granules(coverage, store) self.assertEqual(1, len(granules['features'])) granule_id = name + '.1' self.cat.mosaic_delete_granule(coverage, store, granule_id) granules = self.cat.list_granules(coverage, store) self.assertEqual(0, len(granules['features'])) """ testing external Image mosaic creation """ name = 'cea_mosaic_external' path = 'test/data/mosaic/external' self.cat.create_imagemosaic(name, path, workspace='topp') self.cat._cache.clear() resource = self.cat.get_layer("external").resource self.assertIsNotNone(resource) # add granule to mosaic granule_path = 'test/data/mosaic/granules/cea_20150102.tif' self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(2, len(granules['features'])) # add external granule to mosaic granule_path = os.path.join( os.getcwd(), 'test/data/mosaic/granules/cea_20150103.zip') self.cat.add_granule(granule_path, name, workspace='topp') granules = self.cat.list_granules("external", name, 'topp') self.assertEqual(3, len(granules['features'])) # Delete store store = self.cat.get_store(name) self.cat.delete(store, purge=True, recurse=True) self.cat._cache.clear() def testTimeDimension(self): sf = self.cat.get_workspace("sf") files = shapefile_and_friends( os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date")) self.cat.create_featurestore("boxes_with_end_date", files, sf) get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer( 'boxes_with_end_date').resource # noqa: E501 # configure time as LIST resource = get_resource() timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date") resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual("LIST", timeInfo.presentation) self.assertEqual(True, timeInfo.enabled) self.assertEqual("date", timeInfo.attribute) self.assertEqual("ISO8601", timeInfo.units) # disable time dimension timeInfo = resource.metadata['time'] timeInfo.enabled = False # since this is an xml property, it won't get written unless we modify it resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(False, timeInfo.enabled) # configure with interval, end_attribute and enable again timeInfo.enabled = True timeInfo.presentation = 'DISCRETE_INTERVAL' timeInfo.resolution = '3 days' timeInfo.end_attribute = 'enddate' resource.metadata = {'time': timeInfo} self.cat.save(resource) # and verify resource = get_resource() timeInfo = resource.metadata['time'] self.assertEqual(True, timeInfo.enabled) self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation) self.assertEqual('3 days', timeInfo.resolution_str()) self.assertEqual('enddate', timeInfo.end_attribute)
class GeoserverLayers(object): def __init__(self, gs_apirest, username, password): try: self.__catalog = Catalog(gs_apirest, username=username, password=password) except Exception as err: print("Geoserver API Rest Error: {0}".format(err)) def __createSldStyle(self, sldpath, stylename, ovwrt=False): try: styles_list = [st.name for st in self.__catalog.get_styles()] if ovwrt or not stylename in styles_list: with open(sldpath) as f: self.__catalog.create_style(stylename, f.read(), overwrite=ovwrt) else: print("This style already exists...") except Exception as err: print("Geoserver API Rest Error creating new style: {0}".format(err)) def __rmvResources(self, rsrc): """ Removing resources """ self.__catalog.delete(rsrc) self.__catalog.reload() def rmvStyle(self, stylename): """ Remove style """ try: style = self.__catalog.get_style(stylename) self.__rmvResources(style) except Exception as err: print("Geoserver API Rest Error removing style: {0}".format(err)) def rmvDataStore(self, ds_store): """ Remove DataStore (with his layer) """ try: lay_rm = self.__catalog.get_layer(ds_store) self.__rmvResources(lay_rm) str_rm = self.__catalog.get_store(ds_store) self.__rmvResources(str_rm) except Exception as err: print("Geoserver API Rest Error removing data store: {0}".format(err)) def createGeoserverWMSLayer(self, data, ws_name, ds_name, stylename, sldpath, debug=False): """ Create Geoserver WMS layer Status codes: 2 : "Geoserver layer successfully created" -1 : "Workspace does not exist" -2 : "Datastore already exists" -3 : "Error creating Geoserver layer" -4 : "File missing" """ try: if not self.__catalog.get_workspace(ws_name): print("Workspace does not exist") return -1 ds_list = [i.name for i in self.__catalog.get_stores()] if ds_name in ds_list: print("Datastore already exists") return -2 ds = self.__catalog.create_datastore(ds_name, ws_name) ft = self.__catalog.create_featurestore(ds_name, workspace=ws_name,data=data) print("Geoserver layer successfully created...") self.__createSldStyle(sldpath, stylename) lyr = self.__catalog.get_layer(ds_name) lyr.enabled = True lyr.default_style = stylename self.__catalog.save(lyr) if debug: rsrc = self.__catalog.get_resource(ds_name, workspace=ws_name) print("Data Store XML: {}".format(rsrc.href)) return 2 except Exception as err: print("Geoserver API Rest Error creating new layer: {0}".format(err)) return -3
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD']) def testAbout(self): about_html = self.cat.about() self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testResourcesUpdate(self): res_dest = self.cat.get_resources() count = 0 for rd in res_dest: # only wms layers if rd.resource_type != "wmsLayer": continue # looking for same name ro = self.cat.get_resource(rd.name) if ro is not None: rd.title = ro.title rd.abstract = ro.abstract rd.keywords = ro.keywords rd.projection = ro.projection rd.native_bbox = ro.native_bbox rd.latlon_bbox = ro.latlon_bbox rd.projection_policy = ro.projection_policy rd.enabled = ro.enabled rd.advertised = ro.advertised rd.metadata_links = ro.metadata_links or None self.cat.save(rd) self.cat.reload() # print "Updated layer: " + rd.name count += 1 # print "Total updated layers: " + str(count) def testLayers(self): expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) # Try to create a new Layer Group into the "topp" workspace self.assert_(self.cat.get_workspace("topp") is not None) tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp") self.cat.save(tas2) self.assert_(self.cat.get_layergroup("tasmania_reloaded") is None) self.assert_(self.cat.get_layergroup("tasmania_reloaded", "topp") is not None) tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp") self.assert_("tasmania_reloaded", tas2.name) self.assert_(isinstance(tas2, LayerGroup)) self.assertEqual(tas2.workspace, "topp", tas2.workspace) self.assertEqual(tas2.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas2.layers) self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles) def testStyles(self): self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assert_(self.cat.get_style('non-existing-style') is None) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") try: self.cat.get_store(workspace="best workspace ever", name="best store ever") self.fail('expected exception') except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message) try: self.cat.get_resource(workspace="best workspace ever", store="best store ever", name="best resource ever") except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message)
class ModifyingTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testFeatureTypeSave(self): # test saving round trip rs = self.cat.get_resource("bugsites") old_abstract = rs.abstract new_abstract = "Not the original abstract" enabled = rs.enabled # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(new_abstract, rs.abstract) self.assertEqual(enabled, rs.enabled) # Change keywords on server rs.keywords = ["bugsites", "gsconfig"] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(["bugsites", "gsconfig"], rs.keywords) self.assertEqual(enabled, rs.enabled) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("bugsites") self.assertEqual(old_abstract, rs.abstract) def testDataStoreCreate(self): ds = self.cat.create_datastore("vector_gsconfig") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) def testPublishFeatureType(self): # Use the other test and store creation to load vector data into a database # @todo maybe load directly to database? try: self.testDataStoreCreateAndThenAlsoImportData() except FailedRequestError: pass try: lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) ds = self.cat.get_store("gsconfig_import_test") # make sure it's gone self.assert_(self.cat.get_layer('import') is None) self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326") # and now it's not self.assert_(self.cat.get_layer('import') is not None) finally: # tear stuff down to allow the other test to pass if we run first ds = self.cat.get_store("gsconfig_import_test") lyr = self.cat.get_layer('import') # Delete the existing layer and resource to allow republishing. self.cat.delete(lyr) self.cat.delete(lyr.resource) self.cat.delete(ds) def testDataStoreModify(self): ds = self.cat.get_store("sf") self.assertFalse("foo" in ds.connection_parameters) ds.connection_parameters = ds.connection_parameters ds.connection_parameters["foo"] = "bar" orig_ws = ds.workspace.name self.cat.save(ds) ds = self.cat.get_store("sf") self.assertTrue("foo" in ds.connection_parameters) self.assertEqual("bar", ds.connection_parameters["foo"]) self.assertEqual(orig_ws, ds.workspace.name) @drop_table('import') def testDataStoreCreateAndThenAlsoImportData(self): ds = self.cat.create_datastore("gsconfig_import_test") ds.connection_parameters.update(**DBPARAMS) self.cat.save(ds) ds = self.cat.get_store("gsconfig_import_test") self.cat.add_data_to_store(ds, "import", { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' }) def testCoverageStoreCreate(self): ds = self.cat.create_coveragestore2("coverage_gsconfig") ds.data_url = "file:data/mytiff.tiff" self.cat.save(ds) def testCoverageStoreModify(self): cs = self.cat.get_store("sfdem") self.assertEqual("GeoTIFF", cs.type) cs.type = "WorldImage" self.cat.save(cs) cs = self.cat.get_store("sfdem") self.assertEqual("WorldImage", cs.type) # not sure about order of test runs here, but it might cause problems # for other tests if this layer is misconfigured cs.type = "GeoTIFF" self.cat.save(cs) def testCoverageSave(self): # test saving round trip rs = self.cat.get_resource("Arc_Sample") old_abstract = rs.abstract new_abstract = "Not the original abstract" # # Change abstract on server rs.abstract = new_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(new_abstract, rs.abstract) # Restore abstract rs.abstract = old_abstract self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual(old_abstract, rs.abstract) # Change metadata links on server rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")] enabled = rs.enabled self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEqual( [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")], rs.metadata_links) self.assertEqual(enabled, rs.enabled) srs_before = set(['EPSG:4326']) srs_after = set(['EPSG:4326', 'EPSG:3785']) formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF']) formats_after = set(["PNG", "GIF", "TIFF"]) # set and save request_srs_list self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list)) rs.request_srs_list = rs.request_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list)) # set and save response_srs_list self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list)) rs.response_srs_list = rs.response_srs_list + ['EPSG:3785'] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list)) # set and save supported_formats self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats)) rs.supported_formats = ["PNG", "GIF", "TIFF"] self.cat.save(rs) rs = self.cat.get_resource("Arc_Sample") self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats)) def testWmsStoreCreate(self): ws = self.cat.create_wmsstore("wmsstore_gsconfig") ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" ws.type = "WMS" self.cat.save(ws) def testWmsLayer(self): self.cat.create_workspace("wmstest", "http://example.com/wmstest") wmstest = self.cat.get_workspace("wmstest") wmsstore = self.cat.create_wmsstore("wmsstore", wmstest) wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities" wmsstore.type = "WMS" self.cat.save(wmsstore) wmsstore = self.cat.get_store("wmsstore") self.assertEqual(1, len(self.cat.get_stores(wmstest))) available_layers = wmsstore.get_resources(available=True) for layer in available_layers: new_layer = self.cat.create_wmslayer(wmstest, wmsstore, layer) added_layers = wmsstore.get_resources() self.assertEqual(len(available_layers), len(added_layers)) def testFeatureTypeCreate(self): shapefile_plus_sidecars = shapefile_and_friends("test/data/states") expected = { 'shp': 'test/data/states.shp', 'shx': 'test/data/states.shx', 'dbf': 'test/data/states.dbf', 'prj': 'test/data/states.prj' } self.assertEqual(len(expected), len(shapefile_plus_sidecars)) for k, v in expected.iteritems(): self.assertEqual(v, shapefile_plus_sidecars[k]) sf = self.cat.get_workspace("sf") self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None) self.assertRaises( ConflictingDataError, lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf) ) self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf) ) bogus_shp = { 'shp': 'test/data/Pk50095.tif', 'shx': 'test/data/Pk50095.tif', 'dbf': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } self.assertRaises( UploadError, lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf) ) lyr = self.cat.get_layer("states_test") self.cat.delete(lyr) self.assert_(self.cat.get_layer("states_test") is None) def testCoverageCreate(self): tiffdata = { 'tiff': 'test/data/Pk50095.tif', 'tfw': 'test/data/Pk50095.tfw', 'prj': 'test/data/Pk50095.prj' } sf = self.cat.get_workspace("sf") # TODO: Uploading WorldImage file no longer works??? # ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf) # self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None) # self.assertRaises( # ConflictingDataError, # lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf) # ) self.assertRaises( UploadError, lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf) ) bogus_tiff = { 'tiff': 'test/data/states.shp', 'tfw': 'test/data/states.shx', 'prj': 'test/data/states.prj' } self.assertRaises( UploadError, lambda: self.cat.create_coveragestore("states_raster", bogus_tiff) ) def testLayerSave(self): # test saving round trip lyr = self.cat.get_layer("states") old_attribution = lyr.attribution new_attribution = "Not the original attribution" # change attribution on server lyr.attribution = new_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(new_attribution, lyr.attribution) # Restore attribution lyr.attribution = old_attribution self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(old_attribution, lyr.attribution) self.assertEqual(lyr.default_style.name, "population") old_default_style = lyr.default_style lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next() lyr.styles = [old_default_style] self.cat.save(lyr) lyr = self.cat.get_layer("states") self.assertEqual(lyr.default_style.name, "pophatch") self.assertEqual([s.name for s in lyr.styles], ["population"]) def testStyles(self): # upload new style, verify existence self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Fred", fred.sld_title) # replace style, verify changes self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True) fred = self.cat.get_style("fred") self.assert_(fred is not None) self.assertEqual("Ted", fred.sld_title) # delete style, verify non-existence self.cat.delete(fred, purge=True) self.assert_(self.cat.get_style("fred") is None) # attempt creating new style self.cat.create_style("fred", open("test/fred.sld").read()) fred = self.cat.get_style("fred") self.assertEqual("Fred", fred.sld_title) # verify it can be found via URL and check the name f = self.cat.get_style_by_url(fred.href) self.assert_(f is not None) self.assertEqual(f.name, fred.name) def testWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="blarny") self.assert_(jed is None) jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) jed = self.cat.get_style("topp:jed") self.assert_(jed is not None) self.assertEqual("Fred", jed.sld_title) # replace style, verify changes self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assert_(jed is not None) self.assertEqual("Ted", jed.sld_title) # delete style, verify non-existence self.cat.delete(jed, purge=True) self.assert_(self.cat.get_style("jed", workspace="topp") is None) # attempt creating new style self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp") jed = self.cat.get_style("jed", workspace="topp") self.assertEqual("Fred", jed.sld_title) # verify it can be found via URL and check the full name f = self.cat.get_style_by_url(jed.href) self.assert_(f is not None) self.assertEqual(f.fqn, jed.fqn) def testLayerWorkspaceStyles(self): # upload new style, verify existence self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp") self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp") ned = self.cat.get_style("ned", workspace="topp") zed = self.cat.get_style("zed", workspace="topp") self.assert_(ned is not None) self.assert_(zed is not None) lyr = self.cat.get_layer("states") lyr.default_style = ned lyr.styles = [zed] self.cat.save(lyr) self.assertEqual("topp:ned", lyr.default_style) self.assertEqual([zed], lyr.styles) lyr.refresh() self.assertEqual("topp:ned", lyr.default_style.fqn) self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles]) def testWorkspaceCreate(self): ws = self.cat.get_workspace("acme") self.assertEqual(None, ws) self.cat.create_workspace("acme", "http://example.com/acme") ws = self.cat.get_workspace("acme") self.assertEqual("acme", ws.name) def testWorkspaceDelete(self): self.cat.create_workspace("foo", "http://example.com/foo") ws = self.cat.get_workspace("foo") self.cat.delete(ws) ws = self.cat.get_workspace("foo") self.assert_(ws is None) def testWorkspaceDefault(self): # save orig orig = self.cat.get_default_workspace() neu = self.cat.create_workspace("neu", "http://example.com/neu") try: # make sure setting it works self.cat.set_default_workspace("neu") ws = self.cat.get_default_workspace() self.assertEqual('neu', ws.name) finally: # cleanup and reset to the way things were self.cat.delete(neu) self.cat.set_default_workspace(orig.name) ws = self.cat.get_default_workspace() self.assertEqual(orig.name, ws.name) def testFeatureTypeDelete(self): pass def testCoverageDelete(self): pass def testDataStoreDelete(self): states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) states.enabled = False self.assert_(states.enabled == False) self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == False) states.enabled = True self.cat.save(states) states = self.cat.get_store('states_shapefile') self.assert_(states.enabled == True) def testLayerGroupSave(self): tas = self.cat.get_layergroup("tasmania") self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) tas.layers = tas.layers[:-1] tas.styles = tas.styles[:-1] self.cat.save(tas) # this verifies the local state self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles) # force a refresh to check the remote state tas.refresh() self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers) self.assertEqual(tas.styles, [None, None, None], tas.styles)
def add_to_geoserver(self): # Inspired (copied :) ) from https://groups.google.com/forum/#!msg/geonode-users/R-u57r8aECw/AuEpydZayfIJ # TODO : check license # We connect to the catalog gsUrl = settings.OGC_SERVER['default']['LOCATION'] + "rest" gsUser = settings.OGC_SERVER['default']['USER'] gsPassword = settings.OGC_SERVER['default']['PASSWORD'] cat = Catalog(gsUrl, gsUser, gsPassword) if cat is None: raise Exception('unable to instantiate geoserver catalog') # We get the workspace ws = cat.get_workspace(settings.DEFAULT_WORKSPACE) if ws is None: raise Exception('workspace %s not found in geoserver' % settings.DEFAULT_WORKSPACE) # We get or create the datastore store = cat.get_store(self.datastore_name, ws) if store is None: store = cat.create_datastore(self.datastore_name, ws) store.connection_parameters.update(host="postgres", port="5432", database="postgres", user="******", passwd="postgres", schema='offline_osm', dbtype="postgis") cat.save(store) if store is None: raise Exception('datastore %s not found in geoserver' % self.datastore_name) # We get or create each layer then register it into geonode for layer in Command.layers: layername = layer["name"] self.stdout.write('Adding {} to geoserver...'.format(layername)) layer_exists = (not cat.get_layer(layername) is None) if not layer_exists or not self.options['no_overwrite']: self.stdout.write( ' layer does not exists or no_overwrite unset, we add...') ft = cat.publish_featuretype(layername, store, 'EPSG:4326', srs='EPSG:4326') if ft is None: raise Exception('unable to publish layer %s' % layername) ft.title = 'OpenStreetMap Offline - ' + layername.split( '_')[-1] ft.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.' cat.save(ft) self.stdout.write( ' adding the style for {}...'.format(layername)) # We get or create the workspace style_path = os.path.join(os.path.dirname(__file__), '..', '..', 'styles', layer["style_name"]) if not os.path.exists(style_path): self.stdout.write( ' The file {} does not exist. No style will be applied for {}.' .format(style_path, layername)) else: cat.create_style(layer["style_name"], open(style_path, 'r').read(), overwrite=True, workspace=settings.DEFAULT_WORKSPACE, raw=True) style = cat.get_style(layer["style_name"], ws) if style is None: raise Exception('style not found (%s)' % (layer["style_name"])) publishing = cat.get_layer(layername) if publishing is None: raise Exception('layer not found (%s)' % layerName) publishing.default_style = style cat.save(publishing) self.stdout.write( ' registering {} into geonode...'.format(layername)) resource = cat.get_resource(layername, store, ws) if resource is None: raise Exception('resource not found (%s)' % layername) layer, created = Layer.objects.get_or_create(name=layername) layer.workspace = ws.name layer.store = store.name layer.storeType = store.resource_type layer.typename = "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')) layer.title = resource.title layer.abstract = resource.abstract layer.temporal_extent_start = self.import_timestamp layer.temporal_extent_end = self.import_timestamp try: layer.save() except TypeError as e: # We ignore a specific error in Geonode resourcebase_post_save when settings.SITEURL has no host (relative URL) self.stdout.write( 'TODO : report Geonode exception in resourcebase_post_save when settings.SITEURL has no host (relative URL) : ' + str(e)) pass if created: layer.set_default_permissions() else: self.stdout.write(' layer already exists, we skip.') # We get or create the laygroup self.stdout.write('Adding layergroup to geoserver...') layername = 'offline_osm' layergroup_exists = (not cat.get_layergroup(layername) is None) if not layer_exists or not self.options['no_overwrite']: self.stdout.write( ' layer does not exists or no_overwrite unset, we add...') layergroup = cat.get_layergroup( layername, workspace=settings.DEFAULT_WORKSPACE) if layergroup is None: layergroup = cat.create_layergroup( layername, layers=layernames, workspace=settings.DEFAULT_WORKSPACE) if layergroup is None: raise Exception('unable to publish layer %s' % layername) layergroup.title = 'OpenStreetMap Offline' layergroup.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.' cat.save(layergroup) else: self.stdout.write(' layergroup already exists, we skip.') # TODO : can we add layergroups to Geonode ? self.stdout.write( "laygroup won't be added to geonode (not supported yet)")
def _register_cascaded_layers(service, owner=None): """ Register layers for a cascading WMS """ if service.type == 'WMS' or service.type == "OWS": cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user , _password) # Can we always assume that it is geonode? # Should cascading layers have a separate workspace? cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE) if cascade_ws is None: cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, 'cascade') try: store = cat.get_store(service.name,cascade_ws) except Exception: store = cat.create_wmsstore(service.name, cascade_ws) wms = WebMapService(service.base_url) layers = list(wms.contents) count = 0 for layer in layers: lyr = cat.get_resource(layer, store, cascade_ws) if lyr is None: if service.type in ["WMS","OWS"]: resource = cat.create_wmslayer(cascade_ws, store, layer) elif service.type == "WFS": resource = cat.create_wfslayer(cascade_ws, store, layer) if resource: cascaded_layer, created = Layer.objects.get_or_create( typename = "%s:%s" % (cascade_ws.name, resource.name), defaults = { "name": resource.name, "workspace": cascade_ws.name, "store": store.name, "storeType": store.resource_type, "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": None, "uuid": str(uuid.uuid4()) }) if created: cascaded_layer.save() if cascaded_layer is not None and cascaded_layer.bbox is None: cascaded_layer._populate_from_gs(gs_resource=resource) cascaded_layer.set_default_permissions() service_layer, created = ServiceLayer.objects.get_or_create( service=service, typename=cascaded_layer.name ) service_layer.layer = cascaded_layer service_layer.title=cascaded_layer.title, service_layer.description=cascaded_layer.abstract, service_layer.styles=cascaded_layer.styles service_layer.save() count += 1 else: logger.error("Resource %s from store %s could not be saved as layer" % (layer, store.name)) message = "%d Layers Registered" % count return_dict = {'status': 'ok', 'msg': message } return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200) elif service.type == 'WCS': return HttpResponse('Not Implemented (Yet)', status=501) else: return HttpResponse('Invalid Service Type', status=400)
class Client: def __init__(self, geoserver, username, password): self.restserver = urlparse.urljoin(geoserver, 'rest/') self.wmsserver = urlparse.urljoin(geoserver, 'wms') self.username = username self.password = password self.catalog = Catalog(self.restserver, self.username, self.password) self.tempDir = tempfile.mkdtemp() self.resource = None self.layer = None self.layerName = None logging.basicConfig( format="%(asctime)-15s %(name)-10s %(levelname)-7s : %(message)s", level=logging.WARN) self.logger = logging.getLogger("gsclient") self.logger.setLevel(logging.DEBUG) # setup logging for the gsclient logging.getLogger('pyclowder').setLevel(logging.DEBUG) logging.getLogger('__main__').setLevel(logging.DEBUG) ## this method assume that there is 1 store per layer def getResourceByStoreName(self, storename, workspace): if self.resource != None: self.logger.debug("resource instance found; no need to fetch") return self.resource self.logger.debug("catalog.get_store called") store = self.catalog.get_store(storename, workspace) self.logger.debug("catalog.get_resources called based on store") resources = self.catalog.get_resources(store=store) self.logger.debug("fetched resources from server") if resources == None: return None else: self.resource = resources[0] return self.resource def getLayers(self): layers = self.catalog.get_layers() return layers def getLayerByStoreName(self, storename): self.logger.debug("getLayerbystore name started") layers = self.catalog.get_layers() for layer in layers: if layer.resource.store.name == storename: self.logger.debug("found the layer by store name") return layer return None def getLayerByResource(self, resource): if self.layer != None: self.logger.debug("layer instance found; no need to fetch") return self.layer self.logger.debug("get Layer by Resource started...") layers = self.catalog.get_layers(resource) self.logger.debug("fetched layers from the server") if layers == None: return None else: self.layer = layers[0] return self.layer def mintMetadataWithoutGeoserver(self, workspace, filename, extent): self.logger.debug("Creating wms metadata ... ") metadata = {} wmsLayerName = workspace + ':' + filename metadata['WMS Layer Name'] = wmsLayerName metadata['WMS Service URL'] = self.wmsserver metadata[ 'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng' self.logger.debug('[DONE]') return metadata def mintMetadata(self, workspace, storename, extent): self.logger.debug("Creating wms metadata ... ") metadata = {} layername = None if self.layerName == None: if self.layer == None: self.logger.debug("getResourceByStoreName..") resource = self.getResourceByStoreName(storename, workspace) self.logger.debug("getLayerByResource ...") layer = self.getLayerByResource(resource) #layername = layer.name self.logger.debug("done getting layer name") if layer == None: self.logger.debug('No layer found [DONE]') return metadata else: layername = layer.name else: layername = self.layer.name self.layerName = self.layer.name else: layername = self.layerName # generate metadata wmsLayerName = workspace + ':' + layername metadata['WMS Layer Name'] = wmsLayerName metadata['WMS Service URL'] = self.wmsserver metadata[ 'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng' self.logger.debug('[DONE]') return metadata def uploadShapefile(self, geoserver_url, workspace, storename, filename, projection, secret_key, proxy_on): self.logger.debug("Uploading shapefile" + filename + "...") if (proxy_on.lower() == 'true'): self.logger.debug("proxy set to on ....") # TODO activate proxy_on method if the proxy in clowder works return self.geoserver_manipulation_proxy_off( geoserver_url, workspace, storename, filename, projection) # return self.geoserver_manipulation_proxy_on(geoserver_url, workspace, storename, filename, projection, secret_key) else: return self.geoserver_manipulation_proxy_off( geoserver_url, workspace, storename, filename, projection) def geoserver_manipulation_proxy_off(self, geoserver_url, workspace, storename, filename, projection): self.logger.debug("start geoserver manipulation....") # create workspace if not present is_workspace = False self.logger.debug("checking workspace %s" % workspace) response_worksp = requests.get(self.restserver + 'workspaces/' + workspace, auth=(self.username, self.password)) if response_worksp.status_code != 200: new_worksp = "<workspace><name>" + workspace + "</name></workspace>" response_worksp = requests.post( self.restserver + 'workspaces', headers={"Content-type": "text/xml"}, auth=(self.username, self.password), data=new_worksp) if response_worksp.status_code == 201: is_workspace = True else: is_workspace = True if is_workspace: url = self.restserver + "workspaces/" + workspace + "/datastores/" + storename + "/file.shp" response = None self.logger.debug("put file to geosever %s" % url) with open(filename, 'rb') as f: response = requests.put( url, headers={'content-type': 'application/zip'}, auth=(self.username, self.password), data=f) self.logger.debug(str(response.status_code) + " " + response.text) if response.status_code != 201: self.logger.debug("[DONE]") return False self.set_projection(storename, workspace, projection) return True else: return False def geoserver_manipulation_proxy_on(self, geoserver_url, workspace, storename, filename, projection, secret_key): # create workspace if not present is_workspace = False # this is a direct method, if the proxy works, this should go through proxy last_charactor = geoserver_url[-1] if last_charactor == '/': geoserver_rest = geoserver_url + 'rest' else: geoserver_rest = geoserver_url + '/rest' response_worksp = requests.get(geoserver_rest + '/workspaces/' + workspace + '?key=' + secret_key, auth=(self.username, self.password)) if response_worksp.status_code != 200: new_worksp = "<workspace><name>" + workspace + "</name></workspace>" response_worksp = requests.post( geoserver_rest + '/workspaces' + '?key=' + secret_key, headers={"Content-type": "text/xml"}, auth=(self.username, self.password), data=new_worksp) if response_worksp.status_code == 201: is_workspace = True else: is_workspace = True if is_workspace: url = geoserver_rest + "/workspaces/" + workspace + "/datastores/" + storename + "/file.shp" response = None with open(filename, 'rb') as f: response = requests.put( url + '?key=' + secret_key, headers={'content-type': 'application/zip'}, data=f) self.logger.debug(str(response.status_code) + " " + response.text) if response.status_code != 201: self.logger.debug("[DONE]") return False self.set_projection(storename, workspace, projection) return True else: return False def set_projection(self, storename, workspace, projection): resource = self.getResourceByStoreName(storename, workspace) if resource.projection == None: self.logger.debug('Setting projection' + projection) resource.projection = projection self.catalog.save(resource) self.logger.debug("[DONE]") self.layerName = storename def createThumbnail(self, workspace, storename, extent, width, height): self.logger.debug('Creating Thumbnail ...') layername = None if self.layerName == None: if self.layer == None: self.logger.debug("getResourceByStoreName..") resource = self.getResourceByStoreName(storename, workspace) self.logger.debug("getLayerByResource ...") layer = self.getLayerByResource(resource) self.logger.debug("done getting layer name") if layer == None: self.logger.debug('No layer found [DONE]') return metadata else: layername = layer.name else: self.logger.debug("layer instance found: no need to fetch") layername = self.layer.name self.layerName = self.layer.name else: self.logger.debug("layerName instance found: no need to fetch") layername = self.layerName #wmsLayerName = workspace+":"+layer.name wmsLayerName = workspace + ":" + layername url = self.wmsserver + "?request=GetMap&layers=" + wmsLayerName + "&bbox=" + extent + "&width=" + width + "&height=" + height + "&srs=EPSG:3857&format=image%2Fpng" r = requests.get(url, stream=True) path = os.path.join(self.tempDir, 'tmp.png') if r.status_code == 200: tmp = r.headers['content-disposition'] tmplist = tmp.split(';') for t in tmplist: if t.strip().find('filename=') != -1: path = os.path.join(self.tempDir, t.strip().split('=')[1]) with open(path, 'wb') as f: for chunk in r.iter_content(): f.write(chunk) self.logger.debug('[DONE]') return path else: self.logger.debug('can not create thumbnail [DONE]') return '' def __del__(self): # delete the temp dir if self.tempDir != None: try: import shutil self.logger.debug("Deleting temp dir " + self.tempDir) shutil.rmtree(self.tempDir) self.logger.debug("Deleted Temp file") except OSError as exc: if exc.errno != errno.ENOENT: raise
def get_selected_raster(db,region,variable,date): # logging.info(str(region)+','+str(variable)+','+str(date)) try: # logging.info('Connecting to the database') conn = psycopg2.connect("dbname={0} user={1} host={2} password={3}".format(db,cfg.connection['user'],cfg.connection['host'],cfg.connection['password'])) cur = conn.cursor() storename = db+'_'+region+'_'+variable+'_'+date cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password'],disable_ssl_certificate_validation=True) try: # logging.info('Check if the layer exists') something = cat.get_store(storename,cfg.geoserver['workspace']) if not something: # logging.info('Layer doesnt exist') print "No store" raise Exception else: mean, stddev, min, max = get_vic_summary(db,region, variable, date) # logging.info(str(mean)+str(stddev)+str(min)+str(max)) return storename, mean, stddev, min, max except Exception as e: # logging.info('Entering geoserver code') # logging.error('Error at failed request ' + str(e)) try: # logging.info('Starting the geoserver stuff') sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format(region, variable, date) cur.execute(sql) data = cur.fetchall() # logging.info(str(data)) mean, stddev, min, max = get_vic_summary(db,region, variable, date) # logging.info('Work you piece ...') rest_url = cfg.geoserver['rest_url'] # logging.info(str(rest_url)) if rest_url[-1] != "/": rest_url = rest_url + '/' headers = { 'Content-type': 'image/tiff', } request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format(rest_url, cfg.geoserver['workspace'], storename) # Creating the rest url # logging.info('Get the username and password') user = cfg.geoserver['user'] password = cfg.geoserver['password'] # logging.info('Right before the put command') requests.put(request_url,verify=False,headers=headers, data=data[0][0], auth=(user, password)) # Creating the resource on the geoserver # logging.info(request_url) return storename, mean, stddev, min, max except Exception as er: # logging.info('Error at uplaoding tiff '+ str(e)) return str(er)+' This is while adding the raster layer.' except Exception as err: # logging.info(str(err) + ' This is generic catch all') return str(err)+ ' This is the generic one'
def _register_cascaded_layers(service, owner=None): """ Register layers for a cascading WMS """ if service.type == "WMS" or service.type == "OWS": cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password) # Can we always assume that it is geonode? # Should cascading layers have a separate workspace? cascade_ws = cat.get_workspace(service.name) if cascade_ws is None: cascade_ws = cat.create_workspace(service.name, "cascade") try: store = cat.get_store(service.name, cascade_ws) except Exception: store = cat.create_wmsstore(service.name, cascade_ws) cat.save(store) wms = WebMapService(service.base_url) layers = list(wms.contents) count = 0 for layer in layers: lyr = cat.get_resource(layer, store, cascade_ws) if lyr is None: if service.type in ["WMS", "OWS"]: resource = cat.create_wmslayer(cascade_ws, store, layer) elif service.type == "WFS": resource = cat.create_wfslayer(cascade_ws, store, layer) if resource: bbox = resource.latlon_bbox cascaded_layer, created = Layer.objects.get_or_create( typename="%s:%s" % (cascade_ws.name, resource.name), service=service, defaults={ "name": resource.name, "workspace": cascade_ws.name, "store": store.name, "storeType": store.resource_type, "title": resource.title or "No title provided", "abstract": resource.abstract or "No abstract provided", "owner": None, "uuid": str(uuid.uuid4()), "bbox_x0": bbox[0], "bbox_x1": bbox[1], "bbox_y0": bbox[2], "bbox_y1": bbox[3], }, ) if created: cascaded_layer.save() if cascaded_layer is not None and cascaded_layer.bbox is None: cascaded_layer._populate_from_gs(gs_resource=resource) cascaded_layer.set_default_permissions() service_layer, created = ServiceLayer.objects.get_or_create( service=service, typename=cascaded_layer.name ) service_layer.layer = cascaded_layer service_layer.title = (cascaded_layer.title,) service_layer.description = (cascaded_layer.abstract,) service_layer.styles = cascaded_layer.styles service_layer.save() count += 1 else: logger.error("Resource %s from store %s could not be saved as layer" % (layer, store.name)) message = "%d Layers Registered" % count return_dict = {"status": "ok", "msg": message} return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200) elif service.type == "WCS": return HttpResponse("Not Implemented (Yet)", status=501) else: return HttpResponse("Invalid Service Type", status=400)
def calculate_yield(db,schema): try: conn = psycopg2.connect( "dbname={0} user={1} host={2} password={3}".format(db, cfg.connection['user'], cfg.connection['host'], cfg.connection['password'])) cur = conn.cursor() storename = db+'_'+schema+'_agareas' cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password'],disable_ssl_certificate_validation=True) try: print('Check if the layer exists') something = cat.get_store(storename, cfg.geoserver['workspace']) if not something: print "No store" raise Exception else: print 'Store exists' except Exception as e: print('Entering geoserver code') temp_dir = tempfile.mkdtemp() pg_sql = """SELECT * FROM {0}.agareas""".format(schema) export_pg_table(temp_dir,storename,cfg.connection['host'],cfg.connection['user'],cfg.connection['password'],db,pg_sql) target_zip = os.path.join(os.path.join(temp_dir,storename+'.zip')) with zipfile.ZipFile(target_zip, 'w') as pg_zip: for f in os.listdir(os.path.join(temp_dir)): if '.zip' not in f: f = os.path.join(temp_dir,f) pg_zip.write(f,basename(f)) rest_url = cfg.geoserver['rest_url'] if rest_url[-1] != "/": rest_url = rest_url + '/' headers = { 'Content-type': 'application/zip', } request_url = '{0}workspaces/{1}/datastores/{2}/file.shp'.format(rest_url, cfg.geoserver['workspace'], storename) # Creating the rest url user = cfg.geoserver['user'] password = cfg.geoserver['password'] requests.put(request_url, verify=False, headers=headers, data=open(target_zip,'rb'), auth=(user, password)) # Creating the resource on the geoserver if temp_dir is not None: if os.path.exists(temp_dir): print 'whooo' #shutil.rmtree(temp_dir) sql = """SELECT gid,avg(max) as max FROM(SELECT gid,ensemble,max(gwad) FROM {0}.dssat GROUP BY gid,ensemble ORDER BY gid,ensemble) as foo GROUP BY gid""".format(schema) cur.execute(sql) data = cur.fetchall() data.sort() conn.close() return data,storename except Exception as e: print e return e
class CatalogTests(unittest.TestCase): def setUp(self): self.cat = Catalog("http://localhost:8080/geoserver/rest") def testAbout(self): about_html = self.cat.about() self.assertTrue( '<html xmlns="http://www.w3.org/1999/xhtml"' in about_html) def testGSVersion(self): version = self.cat.gsversion() pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)') self.assertTrue(pat.match('2.2.x')) self.assertTrue(pat.match('2.3.2')) self.assertTrue(pat.match('2.3-SNAPSHOT')) self.assertFalse(pat.match('2.3.y')) self.assertFalse(pat.match('233')) self.assertTrue(pat.match(version)) def testWorkspaces(self): self.assertEqual(7, len(self.cat.get_workspaces())) # marking out test since geoserver default workspace is not consistent # self.assertEqual("cite", self.cat.get_default_workspace().name) self.assertEqual("topp", self.cat.get_workspace("topp").name) def testStores(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") self.assertEqual(9, len(self.cat.get_stores())) self.assertEqual(2, len(self.cat.get_stores(topp))) self.assertEqual(2, len(self.cat.get_stores(sf))) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name) self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name) self.assertEqual("sfdem", self.cat.get_store("sfdem").name) def testResources(self): topp = self.cat.get_workspace("topp") sf = self.cat.get_workspace("sf") states = self.cat.get_store("states_shapefile", topp) sfdem = self.cat.get_store("sfdem", sf) self.assertEqual(19, len(self.cat.get_resources())) self.assertEqual(1, len(self.cat.get_resources(states))) self.assertEqual(5, len(self.cat.get_resources(workspace=topp))) self.assertEqual(1, len(self.cat.get_resources(sfdem))) self.assertEqual(6, len(self.cat.get_resources(workspace=sf))) self.assertEqual("states", self.cat.get_resource("states", states).name) self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name) self.assertEqual("states", self.cat.get_resource("states").name) states = self.cat.get_resource("states") fields = [ states.title, states.abstract, states.native_bbox, states.latlon_bbox, states.projection, states.projection_policy ] self.assertFalse(None in fields, str(fields)) self.assertFalse(len(states.keywords) == 0) self.assertFalse(len(states.attributes) == 0) self.assertTrue(states.enabled) self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name) self.assertEqual("sfdem", self.cat.get_resource("sfdem").name) def testLayers(self): expected = set([ "Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem", "bugsites", "restricted", "streams", "archsites", "roads", "tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries", "tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi", "giant_polygon" ]) actual = set(l.name for l in self.cat.get_layers()) missing = expected - actual extras = actual - expected message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assert_(len(expected ^ actual) == 0, message) states = self.cat.get_layer("states") self.assert_("states", states.name) self.assert_(isinstance(states.resource, ResourceInfo)) self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon'])) self.assertEqual(states.default_style.name, "population") def testLayerGroups(self): expected = set(["tasmania", "tiger-ny", "spearfish"]) actual = set(l.name for l in self.cat.get_layergroups()) missing = expected - actual extras = actual - expected message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % ( extras, missing) self.assert_(len(expected ^ actual) == 0, message) tas = self.cat.get_layergroup("tasmania") self.assert_("tasmania", tas.name) self.assert_(isinstance(tas, LayerGroup)) self.assertEqual(tas.layers, [ 'tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities' ], tas.layers) self.assertEqual(tas.styles, [None, None, None, None], tas.styles) def testStyles(self): self.assertEqual(20, len(self.cat.get_styles())) self.assertEqual("population", self.cat.get_style("population").name) self.assertEqual("popshade.sld", self.cat.get_style("population").filename) self.assertEqual("population", self.cat.get_style("population").sld_name) self.assert_(self.cat.get_style('non-existing-style') is None) def testEscaping(self): # GSConfig is inconsistent about using exceptions vs. returning None # when a resource isn't found. # But the basic idea is that none of them should throw HTTP errors from # misconstructed URLS self.cat.get_style("best style ever") self.cat.get_workspace("best workspace ever") try: self.cat.get_store(workspace="best workspace ever", name="best store ever") self.fail('expected exception') except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message) try: self.cat.get_resource(workspace="best workspace ever", store="best store ever", name="best resource ever") except FailedRequestError, fre: self.assertEqual('No store found named: best store ever', fre.message)
def _register_cascaded_service(url, type, name, username, password, wms=None, owner=None, parent=None): """ Register a service as cascading WMS """ try: service = Service.objects.get(base_url=url) return_dict = {} return_dict['service_id'] = service.pk return_dict['msg'] = "This is an existing Service" return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200) except: # TODO: Handle this error properly pass if wms is None: wms = WebMapService(url) # TODO: Make sure we are parsing all service level metadata # TODO: Handle for setting ServiceProfiletRole service = Service.objects.create(base_url=url, type=type, method='C', name=name, version=wms.identification.version, title=wms.identification.title, abstract=wms.identification.abstract, online_resource=wms.provider.url, owner=owner, parent=parent) service.keywords = ','.join(wms.identification.keywords) service.save() service.set_default_permissions() if type in ['WMS', 'OWS']: # Register the Service with GeoServer to be cascaded cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) cascade_ws = cat.get_workspace(name) if cascade_ws is None: cascade_ws = cat.create_workspace(name, "http://geonode.org/cascade") # TODO: Make sure there isn't an existing store with that name, and # deal with it if there is try: cascade_store = cat.get_store(name, cascade_ws) except: cascade_store = cat.create_wmsstore(name, cascade_ws, username, password) cascade_store.capabilitiesURL = url cascade_store.type = "WMS" cat.save(cascade_store) available_resources = cascade_store.get_resources(available=True) elif type == 'WFS': # Register the Service with GeoServer to be cascaded cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) # Can we always assume that it is geonode? cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE) if cascade_ws is None: cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, "http://geonode.org/cascade") try: wfs_ds = cat.get_store(name, cascade_ws) except: wfs_ds = cat.create_datastore(name, cascade_ws) connection_params = { "WFSDataStoreFactory:MAXFEATURES": "0", "WFSDataStoreFactory:TRY_GZIP": "true", "WFSDataStoreFactory:PROTOCOL": "false", "WFSDataStoreFactory:LENIENT": "true", "WFSDataStoreFactory:TIMEOUT": "3000", "WFSDataStoreFactory:BUFFER_SIZE": "10", "WFSDataStoreFactory:ENCODING": "UTF-8", "WFSDataStoreFactory:WFS_STRATEGY": "nonstrict", "WFSDataStoreFactory:GET_CAPABILITIES_URL": url, } if username and password: connection_params["WFSDataStoreFactory:USERNAME"] = username connection_params["WFSDataStoreFactory:PASSWORD"] = password wfs_ds.connection_parameters = connection_params cat.save(wfs_ds) available_resources = wfs_ds.get_resources(available=True) # Save the Service record service, created = Service.objects.get_or_create(type=type, method='C', base_url=url, name=name, owner=owner) service.save() service.set_default_permissions() elif type == 'WCS': return HttpResponse('Not Implemented (Yet)', status=501) else: return HttpResponse('Invalid Method / Type combo: ' + 'Only Cascaded WMS, WFS and WCS supported', mimetype="text/plain", status=400) message = "Service %s registered" % service.name return_dict = [{ 'status': 'ok', 'msg': message, 'service_id': service.pk, 'service_name': service.name, 'service_title': service.title, 'available_layers': available_resources }] if settings.USE_QUEUE: # Create a layer import job WebServiceHarvestLayersJob.objects.get_or_create(service=service) else: _register_cascaded_layers(service) return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200)
#!/usr/bin/env python """ gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API. The project is distributed under a MIT License . """ __author__ = "David Winslow" __copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans" __license__ = "MIT" from geoserver.catalog import Catalog cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver") name = 'foo' ds = cat.create_datastore(name) ds.connection_parameters.update(host="localhost", port="5432", database="gis", user="******", passwd="", dbtype="postgis") cat.save(ds) ds = cat.get_store(name) components = \ dict((ext, "myfile." + ext) for ext in ["shp", "prj", "shx", "dbf"]) cat.add_data_to_store(ds, "mylayer", components)
def _register_cascaded_layers(service, owner=None): """ Register layers for a cascading WMS """ if service.type == 'WMS' or service.type == "OWS": cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest", _user, _password) # Can we always assume that it is geonode? # Should cascading layers have a separate workspace? cascade_ws = cat.get_workspace(service.name) if cascade_ws is None: cascade_ws = cat.create_workspace(service.name, 'cascade') try: store = cat.get_store(service.name, cascade_ws) except Exception: store = cat.create_wmsstore(service.name, cascade_ws) cat.save(store) wms = WebMapService(service.base_url) layers = list(wms.contents) count = 0 for layer in layers: lyr = cat.get_resource(layer, store, cascade_ws) if lyr is None: if service.type in ["WMS", "OWS"]: resource = cat.create_wmslayer(cascade_ws, store, layer) elif service.type == "WFS": resource = cat.create_wfslayer(cascade_ws, store, layer) if resource: bbox = resource.latlon_bbox cascaded_layer, created = Layer.objects.get_or_create( typename="%s:%s" % (cascade_ws.name, resource.name), service=service, defaults={ "name": resource.name, "workspace": cascade_ws.name, "store": store.name, "storeType": store.resource_type, "title": resource.title or 'No title provided', "abstract": resource.abstract or 'No abstract provided', "owner": None, "uuid": str(uuid.uuid4()), "bbox_x0": bbox[0], "bbox_x1": bbox[1], "bbox_y0": bbox[2], "bbox_y1": bbox[3], }) if created: cascaded_layer.save() if cascaded_layer is not None and cascaded_layer.bbox is None: cascaded_layer._populate_from_gs( gs_resource=resource) cascaded_layer.set_default_permissions() service_layer, created = ServiceLayer.objects.get_or_create( service=service, typename=cascaded_layer.name) service_layer.layer = cascaded_layer service_layer.title = cascaded_layer.title, service_layer.description = cascaded_layer.abstract, service_layer.styles = cascaded_layer.styles service_layer.save() count += 1 else: logger.error( "Resource %s from store %s could not be saved as layer" % (layer, store.name)) message = "%d Layers Registered" % count return_dict = {'status': 'ok', 'msg': message} return HttpResponse(json.dumps(return_dict), mimetype='application/json', status=200) elif service.type == 'WCS': return HttpResponse('Not Implemented (Yet)', status=501) else: return HttpResponse('Invalid Service Type', status=400)