def delete_map(request):
map_id = request.POST.get('map_id', False)
cat = Catalog(map_url, 'admin', 'geoserver')
if cat.get_layer('Map:' + map_id):
cat.delete(cat.get_layer('Map:' + map_id))
cat.reload()
for label_type in range(1, 8):
if cat.get_layer('Mask:' + map_id + '_' + str(label_type)):
cat.delete(cat.get_layer('Mask:' + map_id + '_' + str(label_type)))
cat.reload()
try:
if cat.get_store(map_id):
st = cat.get_store(map_id)
cat.delete(st)
cat.reload()
except Exception:
pass
if Bmap.objects.filter(id=map_id):
map_name = Bmap.objects.get(id=map_id).name
dir_root = os.path.join(MAPBASEPATH, map_name)
delete_files = (map_id + ".jpg", 'chaneltransform.tif',
'chaneltransform_rpc.txt', 'chaneltransformRPC.txt',
'chaneltransformRPC.tif.ovr', 'label.tif',
'label_rpc.txt', 'labelRPC.tif')
for file in delete_files:
if os.path.exists(file):
os.remove(os.path.join(dir_root, file))
Bmap.objects.filter(id=map_id).delete()
return HttpResponse("success")
def addLayersToGeoserver(self, options):
cat = Catalog(self.geoserver_rest_url, options["geoserveradmin"], options["gpw"])
try:
ds = cat.get_store(options["alias"])
except Exception as e:
raise Exception("Erreur de récupération du workspace")
layers = []
try:
# connect to tables and create layers and correct urban styles
for table in self.urb:
try:
style = self.urb[table]
ft = cat.publish_featuretype(table, ds, "EPSG:31370", srs="EPSG:31370")
gs_style = cat.get_style(style)
cat.save(ft)
res_name = ft.dirty["name"]
res_title = options["alias"] + "_" + table
cat.save(ft)
layer_name = ds.workspace.name + ":" + res_name
new_layer = cat.get_layer(layer_name)
new_layer.default_style = gs_style
cat.save(new_layer)
layers.append({"res_name": res_name, "res_title": res_title})
except Exception as e:
# a verifier une fois un possesion des styles
print(str(e))
except Exception as e:
print(str(e))
raise Exception("Erreur lors de la récupération des couches depuis Geoserver")
return layers
def main():
parser = argparse.ArgumentParser(
description="OWS12 Sentinel harvest granules script.")
parser.add_argument('-c',
'--catalog',
nargs=3,
help='Geoserver REST URL and authentication',
default=('http://localhost:8080/geoserver/rest',
'admin', 'geoserver'))
parser.add_argument(
'-s',
'--store',
nargs=1,
help='The store name to retrieve coverages with workspace name '
'(ows12:landsat)')
parser.add_argument(
'-i',
'--insert',
action="store_true",
default=False,
help='Harvest (insert) new granule on a given mosaic store')
parser.add_argument('-d',
'--delete',
nargs=1,
type=int,
help='Delete granules older than given months value.')
parser.add_argument('granules', help='Path to ingestion file.')
args = parser.parse_args()
sentinel = SentinelSat()
# Before we proceed, check if we retrieved any files during previous processing step
if not os.path.exists(
os.path.join(args.granules, sentinel.processed_granules)):
logger.info(
'Missing file from previous download job.\nPlease run this job first!'
)
sys.exit(1)
num_lines = sum(1 for l in open(
os.path.join(args.granules, sentinel.processed_granules)))
if num_lines == 0:
logger.info('Skipping processing step. No files to process found!')
sys.exit(0)
catalog = Catalog(service_url=args.catalog[0],
username=args.catalog[1],
password=args.catalog[2])
store = catalog.get_store(args.store[0].split(':')[1],
args.store[0].split(':')[0])
coverages = catalog.mosaic_coverages(store)
# Harvest granules
if args.insert:
sentinel.insert_granules(args.granules, catalog, coverages, store)
# Delete granules
if args.delete:
sentinel.delete_granules(catalog, coverages, store, args.delete)
def addLayersToGeoserver(self, options):
cat = Catalog(self.geoserver_rest_url, options['geoserveradmin'], options['gpw'])
try:
ds = cat.get_store(options['alias'])
except Exception as e:
raise Exception('Erreur de récupération du workspace')
layers = []
try:
#connect to tables and create layers and correct urban styles
for table in self.urb:
try:
style = self.urb[table]
ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370')
gs_style = cat.get_style(style)
cat.save(ft)
res_name = ft.dirty['name']
res_title = options['alias']+"_"+table
cat.save(ft)
layer_name = ds.workspace.name + ':' + res_name
new_layer = cat.get_layer(layer_name)
new_layer.default_style = gs_style
cat.save(new_layer)
layers.append({ 'res_name' : res_name, 'res_title' : res_title })
except Exception as e:
# a verifier une fois un possesion des styles
print(str(e))
except Exception as e:
print(str(e))
raise Exception('Erreur lors de la récupération des couches depuis Geoserver')
return layers
def main(options):
#connect to geoserver
cat = Catalog("http://localhost:8080/geoserver/rest", "admin", options.gpw)
#create datrastore for URB schema
ws = cat.create_workspace(options.alias,'imio.be')
ds = cat.create_datastore(options.alias, ws)
ds.connection_parameters.update(
host=options.urbanUrl,
port="5432",
database=options.database,
user="******",
passwd=options.ropw,
dbtype="postgis")
cat.save(ds)
ds = cat.get_store(options.alias)
#config object
urb = {
"capa":"Parcelles",
"toli":"cadastre_ln_toponymiques",
"canu":"cadastre_pt_num",
"cabu":"Batiments",
"gept":"cadastre_points_generaux",
"gepn":"cadastre_pol_gen",
"inpt":"point",
"geli":"cadastre_ln_generales",
"inli":"cadastre_ln_informations",
"topt":"point",
}
#connect to tables and create layers and correct urban styles
for table in urb:
style = urb[table]
ft = cat.publish_featuretype(table, ds, 'EPSG:31370', srs='EPSG:31370')
ft.default_style = style
cat.save(ft)
resource = ft.resource
resource.title = options.alias+"_"+table
resource.save()
layer, created = Layer.objects.get_or_create(name=layerName, defaults={
"workspace": ws.name,
"store": ds.name,
"storeType": ds.resource_type,
"typename": "%s:%s" % (ws.name.encode('utf-8'), resource.name.encode('utf-8')),
"title": resource.title or 'No title provided',
"abstract": resource.abstract or 'No abstract provided',
#"owner": owner,
"uuid": str(uuid.uuid4()),
"bbox_x0": Decimal(resource.latlon_bbox[0]),
"bbox_x1": Decimal(resource.latlon_bbox[1]),
"bbox_y0": Decimal(resource.latlon_bbox[2]),
"bbox_y1": Decimal(resource.latlon_bbox[3])
})
set_attributes(layer, overwrite=True)
if created: layer.set_default_permissions()
def add_layer(shp):
global ds_name, ws_name, appserver
global instance
cat = Catalog(appserver + '/rest')
ws = cat.get_workspace(ws_name)
ds = cat.get_store(ds_name, ws_name)
print 'starting {0}...'.format(shp)
components = dict((ext, shp + '.' + ext) for ext in ['shp', 'prj', 'shx', 'dbf'])
cat.add_data_to_store(ds, shp, components, ws, True)
print '{0} finished ...'.format(shp)
def _create_geoserver_geonode_layer(new_table, sld_type, title, the_user,
permissions):
# Create the Layer in GeoServer from the table
try:
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
settings.OGC_SERVER['default']['USER'],
settings.OGC_SERVER['default']['PASSWORD'])
ds = cat.get_store("uploaded") # name of store in WFP-Geonode
cat.publish_featuretype(new_table, ds, "EPSG:4326", srs="EPSG:4326")
# changing the layer's title in the title set by the user
resource = cat.get_resource(new_table, workspace="geonode")
resource.title = title
cat.save(resource)
except Exception as e:
msg = "Error creating GeoServer layer for %s: %s" % (new_table, str(e))
print msg
return None, msg
# Create the Layer in GeoNode from the GeoServer Layer
try:
link_to_sld = "{location}styles/{sld_type}".format(
**{
'location': settings.OGC_SERVER['default']['LOCATION'],
'sld_type': sld_type
})
r = requests.get(link_to_sld)
sld = r.text
sld = sld.replace(
"name_of_layer", new_table
) # "name_of_layer" is set in the predefined sld in geoserver (polygon_style, line_style, point_style)
cat.create_style(new_table, sld, overwrite=True)
style = cat.get_style(new_table)
layer = cat.get_layer(new_table)
layer.default_style = style
cat.save(layer)
gs_slurp(owner=the_user, filter=new_table)
from geonode.base.models import ResourceBase
layer = ResourceBase.objects.get(title=title)
geoserver_post_save(layer, ResourceBase)
# assign permissions for this layer
permissions_dict = json.loads(permissions) # needs to be dictionary
if permissions_dict is not None and len(permissions_dict.keys()) > 0:
layer.set_permissions(permissions_dict)
except Exception as e:
msg = "Error creating GeoNode layer for %s: %s" % (new_table, str(e))
return None, msg
def handle_geoserver_layers(view_names,
store_name,
db_params,
schema_name,
logger=print):
"""Publish database views as GeoServer layers
Parameters
----------
view_pairs: list
An iterable of two-element tuples with the names of the
``coarse`` and ``detail`` database views that are to be published
as GeoServer layers
store_name: str
Name of the GeoServer store to use. This store will be created in case
it does not exist
db_params: dict
A mapping with parameters used for connecting to the database
schema_name: str
Name of the database schema where the views to publish reside
logger: function, optional
A function that is used to output information.
Returns
-------
list
An iterable with the geoserver layers that have been published
"""
gs_catalog = Catalog(
service_url=settings.OGC_SERVER["default"]["LOCATION"] + "rest",
username=settings.OGC_SERVER["default"]["USER"],
password=settings.OGC_SERVER["default"]["PASSWORD"])
logger("Retrieving geoserver workspace...")
workspace = get_geoserver_workspace(gs_catalog)
store = gs_catalog.get_store(store_name, workspace=workspace)
if store is None:
logger("Creating geoserver store...")
store = get_postgis_store(gs_catalog, store_name, workspace, db_params,
schema_name)
layers = []
for view_name in view_names:
logger("Adding {!r} as a geoserver layer...".format(view_name))
featuretype = gs_catalog.publish_featuretype(view_name,
store,
"EPSG:4326",
srs="EPSG:4326")
gs_catalog.save(featuretype)
layers.append(featuretype)
return layers
class GSAddMosaicGranule(BaseOperator):
@apply_defaults
def __init__(self, granule_abs_path, geoserver_rest_url, gs_user,
gs_password, imagemosaic_storename, *args, **kwargs):
self.granule_abs_path = granule_abs_path
self.catalog = Catalog(geoserver_rest_url, gs_user, gs_password)
self.store_name = imagemosaic_storename
super(GSAddMosaicGranule, self).__init__(*args, **kwargs)
def execute(self, context):
log.info("GSAddMosaicGranule params list")
log.info('Mosaic granule: %s', self.granule_abs_path)
store = self.catalog.get_store(self.store_name)
self.catalog.harvest_externalgranule('file://' + self.granule_abs_path,
store)
def handle_geoserver_layer(table_name, store_name, db_params, schema_name,
default_style_name="", logger=print):
"""Publish a database table/view as a GeoServer layer
Parameters
----------
store_name: str
Name of the GeoServer store to use. This store will be created in case
it does not exist
db_params: dict
A mapping with parameters used for connecting to the database
schema_name: str
Name of the database schema where the views to publish reside
logger: function, optional
A function that is used to output information.
Returns
-------
list
An iterable with the geoserver layers that have been published
"""
logger("inside handle_geoserver_layer: {}".format(locals()))
gs_catalog = Catalog(
service_url=settings.OGC_SERVER["default"]["LOCATION"] + "rest",
username=settings.OGC_SERVER["default"]["USER"],
password=settings.OGC_SERVER["default"]["PASSWORD"]
)
logger("Retrieving geoserver workspace...")
workspace = get_geoserver_workspace(gs_catalog)
store = gs_catalog.get_store(store_name, workspace=workspace)
if store is None:
logger("Creating geoserver store...")
store = get_postgis_store(gs_catalog, store_name, workspace, db_params,
schema_name)
logger("Adding {!r} as a geoserver layer...".format(table_name))
featuretype = gs_catalog.publish_featuretype(
table_name, store, "EPSG:4326", srs="EPSG:4326")
logger("dir(featuretype): {}".format(dir(featuretype)))
layer = gs_catalog.get_layer(featuretype.name)
if default_style_name != "":
logger("Setting default style for layer...")
layer._set_default_style(default_style_name)
gs_catalog.save(featuretype)
gs_catalog.save(layer)
return featuretype
def pre_delete_service(instance, sender, **kwargs):
for layer in instance.layer_set.all():
layer.delete()
# if instance.method == 'H':
# gn = Layer.objects.gn_catalog
# gn.control_harvesting_task('stop', [instance.external_id])
# gn.control_harvesting_task('remove', [instance.external_id])
if instance.method == 'C':
try:
_user = settings.OGC_SERVER['default']['USER']
_password = settings.OGC_SERVER['default']['PASSWORD']
gs = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user , _password)
cascade_store = gs.get_store(instance.name, settings.CASCADE_WORKSPACE)
gs.delete(cascade_store, recurse=True)
except FailedRequestError:
logger.error("Could not delete cascading WMS Store for %s - maybe already gone" % instance.name)
def createLayer():
cat = Catalog('http://localhost:8081/geoserver/rest/', 'admin',
'geoserver')
store = cat.get_store('test')
geom = JDBCVirtualTableGeometry('newgeom', 'LineString', '4326')
ft_name = 'my_jdbc_vt_test'
epsg_code = 'EPSG:4326'
sql = 'select ST_MakeLine(wkb_geometry ORDER BY waypoint) As newgeom, assetid, runtime from waypoints group by assetid,runtime'
keyColumn = None
parameters = None
jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn,
parameters)
ft = cat.publish_featuretype(ft_name,
store,
epsg_code,
jdbc_virtual_table=jdbc_vt)
def pre_delete_service(instance, sender, **kwargs):
for layer in instance.layer_set.all():
layer.delete()
# if instance.method == 'H':
# gn = Layer.objects.gn_catalog
# gn.control_harvesting_task('stop', [instance.external_id])
# gn.control_harvesting_task('remove', [instance.external_id])
if instance.method == 'C':
try:
_user = settings.OGC_SERVER['default']['USER']
_password = settings.OGC_SERVER['default']['PASSWORD']
gs = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user, _password)
cascade_store = gs.get_store(
instance.name, settings.CASCADE_WORKSPACE)
gs.delete(cascade_store, recurse=True)
except FailedRequestError:
logger.error(
"Could not delete cascading WMS Store for %s - maybe already gone" % instance.name)
class GSAddMosaicGranule(BaseOperator):
@apply_defaults
def __init__(self,
get_inputs_from,
geoserver_rest_url,
gs_user,
gs_password,
store_name,
workspace=None,
*args,
**kwargs):
self.get_inputs_from = get_inputs_from
self.catalog = Catalog(geoserver_rest_url, gs_user, gs_password)
self.store_name = store_name
self.workspace = workspace
log.info(
'--------------------GeoServer_PLUGIN Add granule------------')
super(GSAddMosaicGranule, self).__init__(*args, **kwargs)
def execute(self, context):
input_paths = context["task_instance"].xcom_pull(self.get_inputs_from,
key=XCOM_RETURN_KEY)
if input_paths is None:
log.info("Nothing to process")
return None
# If message from XCom is a string with single file path, turn it into a string
if isinstance(input_paths, six.string_types):
input_paths = [input_paths]
for granule in input_paths:
store = self.catalog.get_store(self.store_name,
workspace=self.workspace)
if store is None:
log.info("GeoServer Store {} is not accessible!".format(
self.store_name))
return None
granule = 'file://' + granule
log.info('Mosaic granule: %s', granule)
self.catalog.add_granule(granule, store, workspace=self.workspace)
class GSAddMosaicGranule(BaseOperator):
@apply_defaults
def __init__(self, geoserver_rest_url, gs_user, gs_password,
imagemosaic_storename, mosaic_path, index, *args, **kwargs):
self.catalog = Catalog(geoserver_rest_url, gs_user, gs_password)
self.store_name = imagemosaic_storename
self.mosaic_path = mosaic_path
self.index = index
log.info('--------------------GDAL_PLUGIN Add granule------------')
super(GSAddMosaicGranule, self).__init__(*args, **kwargs)
def execute(self, context):
task_instance = context['task_instance']
granule = task_instance.xcom_pull('rsync_' + str(self.index),
key=xk.GRANULE_TO_UPLOAD_PREFIX +
str(self.index))
log.info("GSAddMosaicGranule params list")
log.info('Mosaic granule: %s', granule)
store = self.catalog.get_store(self.store_name)
granule = 'file://' + self.mosaic_path + '/' + granule
log.info(granule)
self.catalog.harvest_externalgranule(granule, store)
def deleteGeoserverStore(storeName,
workspace=workspace,
purge=None,
recurse=True):
"""
Delete Geoserver Data Store.
Args:
storeName
Kwargs (Default):
workspace(geocolorado)
purge(None) - if purge is True. Data files will be deleted.
recurse (True) - Store and all metadata items will be deleted.
"""
cat = Catalog("{0}/rest/".format(geoserver_connection), geoserver_username,
geoserver_password)
ws = cat.get_workspace(workspace)
ds = cat.get_store(storeName, workspace=ws)
cat.delete(ds, purge=purge, recurse=recurse)
msg = "metadata and data files removed." if purge else "only metadata items removed."
return "DataStore: {0} deleted from geoServer with {1}".format(
storeName, msg)
def remove_geoserver_layer(self, storename, layername):
self.logger.debug("start removing the geoserver layer")
last_charactor = self.geoServer[-1]
if last_charactor == '/':
geoserver_rest = self.geoServer + 'rest'
else:
geoserver_rest = self.geoServer + '/rest'
cat = Catalog(geoserver_rest,
username=self.gs_username,
password=self.gs_password)
# worksp = cat.get_workspace(gs_workspace)
store = cat.get_store(storename)
self.logger.debug("store name %s" % store)
layer = cat.get_layer(layername)
self.logger.debug("layer name %s" % layer)
try:
self.logger.debug("deleting the layer...")
cat.delete(layer)
cat.reload()
self.logger.debug("deleting the store...")
cat.delete(store)
cat.reload
except Exception:
self.logger.error("Failed to remove from geoserver")
def publishGeoserver(appdef, progress):
viewCrs = appdef["Settings"]["App view CRS"]
usesGeoServer = False
for applayer in appdef["Layers"]:
if applayer.method != METHOD_FILE:
if applayer.layer.type() == applayer.layer.VectorLayer and applayer.layer.providerType().lower() != "wfs":
usesGeoServer = True
if not usesGeoServer:
return
progress.setText("Publishing to GeoServer")
progress.setProgress(0)
geoserverUrl = appdef["Deploy"]["GeoServer url"] + "/rest"
geoserverPassword = appdef["Deploy"]["GeoServer password"]
geoserverUsername = appdef["Deploy"]["GeoServer username"]
workspaceName = appdef["Deploy"]["GeoServer workspace"]
dsName = "ds_" + workspaceName
host = appdef["Deploy"]["PostGIS host"]
port = appdef["Deploy"]["PostGIS port"]
postgisUsername = appdef["Deploy"]["PostGIS username"]
postgisPassword = appdef["Deploy"]["PostGIS password"]
database = appdef["Deploy"]["PostGIS database"]
schema = appdef["Deploy"]["PostGIS schema"]
catalog = Catalog(geoserverUrl, geoserverUsername, geoserverPassword)
workspace = catalog.get_workspace(workspaceName)
if workspace is None:
workspace = catalog.create_workspace(workspaceName, workspaceName)
try:
store = catalog.get_store(dsName, workspace)
resources = store.get_resources()
for resource in resources:
layers = catalog.get_layers(resource)
for layer in layers:
catalog.delete(layer)
catalog.delete(resource)
catalog.delete(store)
except Exception:
pass
try:
store = catalog.get_store(dsName, workspace)
except FailedRequestError:
store = None
for i, applayer in enumerate(appdef["Layers"]):
layer = applayer.layer
if applayer.method != METHOD_FILE and applayer.method != METHOD_DIRECT:
name = safeName(layer.name())
sld, icons = getGsCompatibleSld(layer)
if sld is not None:
catalog.create_style(name, sld, True)
uploadIcons(icons, geoserverUsername, geoserverPassword, catalog.gs_base_url)
if layer.type() == layer.VectorLayer:
if applayer.method == METHOD_WFS_POSTGIS or applayer.method == METHOD_WMS_POSTGIS:
if store is None:
store = catalog.create_datastore(dsName, workspace)
store.connection_parameters.update(
host=host, port=str(port), database=database, user=postgisUsername, schema=schema,
passwd=postgisPassword, dbtype="postgis")
catalog.save(store)
catalog.publish_featuretype(name, store, layer.crs().authid())
else:
path = getDataFromLayer(layer, viewCrs)
catalog.create_featurestore(name,
path,
workspace=workspace,
overwrite=True)
gslayer = catalog.get_layer(name)
r = gslayer.resource
r.dirty['srs'] = viewCrs
catalog.save(r)
elif layer.type() == layer.RasterLayer:
path = getDataFromLayer(layer, viewCrs)
catalog.create_coveragestore(name,
path,
workspace=workspace,
overwrite=True)
if sld is not None:
publishing = catalog.get_layer(name)
publishing.default_style = catalog.get_style(name)
catalog.save(publishing)
progress.setProgress(int((i+1)*100.0/len(appdef["Layers"])))
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assert_(self.cat.get_layer('import') is None)
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assert_(self.cat.get_layer('import') is not None)
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
try:
if lyr:
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
if ds:
self.cat.delete(ds)
except:
pass
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
@drop_table('import2')
def testVirtualTables(self):
ds = self.cat.create_datastore("gsconfig_import_test2")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test2")
self.cat.add_data_to_store(ds, "import2", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
store = self.cat.get_store("gsconfig_import_test2")
geom = JDBCVirtualTableGeometry('the_geom','MultiPolygon','4326')
ft_name = 'my_jdbc_vt_test'
epsg_code = 'EPSG:4326'
sql = "select * from import2 where 'STATE_NAME' = 'Illinois'"
keyColumn = None
parameters = None
jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn, parameters)
ft = self.cat.publish_featuretype(ft_name, store, epsg_code, jdbc_virtual_table=jdbc_vt)
# DISABLED; this test works only in the very particular case
# "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR
# def testCoverageStoreCreate(self):
# ds = self.cat.create_coveragestore2("coverage_gsconfig")
# ds.data_url = "file:test/data/mytiff.tiff"
# self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
# sanitize the layer name - validation will fail on newer geoservers
name = layer.replace(':', '_')
new_layer = self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
changed_layer = added_layers[0]
self.assertEqual(True, changed_layer.advertised)
self.assertEqual(True, changed_layer.enabled)
changed_layer.advertised = False
changed_layer.enabled = False
self.cat.save(changed_layer)
self.cat._cache.clear()
changed_layer = wmsstore.get_resources()[0]
changed_layer.fetch()
self.assertEqual(False, changed_layer.advertised)
self.assertEqual(False, changed_layer.enabled)
# Testing projection and projection policy changes
changed_layer.projection = "EPSG:900913"
changed_layer.projection_policy = "REPROJECT_TO_DECLARED"
self.cat.save(changed_layer)
self.cat._cache.clear()
layer = self.cat.get_layer(changed_layer.name)
self.assertEqual(layer.resource.projection_policy, changed_layer.projection_policy)
self.assertEqual(layer.resource.projection, changed_layer.projection)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
ft_ext = self.cat.create_coveragestore_external_geotiff("Pk50095_ext", 'file:test/data/Pk50095.tif', sf)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = { 'title': 'Not the original attribution',
'width': '123',
'height': '321',
'href': 'http://www.georchestra.org',
'url': 'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg',
'type': 'image/jpeg' }
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# check count before tests (upload)
count = len(self.cat.get_styles())
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assert_(f is not None)
self.assertEqual(f.name, fred.name)
# compare count after upload
self.assertEqual(count +1, len(self.cat.get_styles()))
# attempt creating a new style without "title"
self.cat.create_style("notitle", open("test/notitle.sld").read())
notitle = self.cat.get_style("notitle")
self.assertEqual(None, notitle.sld_title)
def testWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assert_(jed is None)
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assert_(self.cat.get_style("jed", workspace="topp") is None)
# attempt creating new style
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assert_(f is not None)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp")
self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assert_(ned is not None)
self.assert_(zed is not None)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def testImageMosaic(self):
"""
Test case for Issue #110
"""
# testing the mosaic creation
name = 'cea_mosaic'
data = open('test/data/mosaic/cea.zip', 'rb')
self.cat.create_imagemosaic(name, data)
# get the layer resource back
self.cat._cache.clear()
resource = self.cat.get_layer(name).resource
self.assert_(resource is not None)
# delete granule from mosaic
coverage = name
store = self.cat.get_store(name)
granule_id = name + '.1'
self.cat.mosaic_delete_granule(coverage, store, granule_id)
def testTimeDimension(self):
sf = self.cat.get_workspace("sf")
files = shapefile_and_friends(os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date"))
self.cat.create_featurestore("boxes_with_end_date", files, sf)
get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer('boxes_with_end_date').resource
# configure time as LIST
resource = get_resource()
timeInfo = DimensionInfo("time", "true", "LIST", None, "ISO8601", None, attribute="date")
resource.metadata = {'time':timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual("date", timeInfo.attribute)
self.assertEqual("ISO8601", timeInfo.units)
# disable time dimension
timeInfo = resource.metadata['time']
timeInfo.enabled = False
# since this is an xml property, it won't get written unless we modify it
resource.metadata = {'time' : timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(False, timeInfo.enabled)
# configure with interval, end_attribute and enable again
timeInfo.enabled = True
timeInfo.presentation = 'DISCRETE_INTERVAL'
timeInfo.resolution = '3 days'
timeInfo.end_attribute = 'enddate'
resource.metadata = {'time' : timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(True, timeInfo.enabled)
self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation)
self.assertEqual('3 days', timeInfo.resolution_str())
self.assertEqual('enddate', timeInfo.end_attribute)
ds_name = 'ne_pg'
lg_name = "ne_pg"
cat = Catalog(appserver + '/rest')
# check if workspace exists, bail if it does (safter than deleting it)
if any(ws.name == ws_name for ws in cat.get_workspaces()):
print 'workspace already exists...'
ws = cat.get_workspace(ws_name)
else:
print 'creating workspace...'
ws = cat.create_workspace(ws_name, 'http://www.naturalearth.org')
if any(ds.workspace.name == ws.name and ds.name == ds_name for ds in cat.get_stores()):
print 'datastore already exists...'
ds = cat.get_store(ds_name, ws_name)
else:
print 'creating datastore'
ds = cat.create_datastore(ds_name, ws.name)
ds.connection_parameters.update(
host=dbserver,
port='5432',
database='ne_tmp',
user='******',
password='******',
dbtype='postgis')
cat.save(ds)
ds = cat.get_store(ds_name, ws_name)
layers = []
styles = []
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'],
username=GSPARAMS['GSUSER'],
password=GSPARAMS['GSPASSWORD'])
def testAbout(self):
about_html = self.cat.about()
self.assertTrue(
'<html xmlns="http://www.w3.org/1999/xhtml"' in str(about_html))
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d+(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertTrue(pat.match('2.10.1'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
self.assertEqual("topp", self.cat.get_workspaces("topp")[-1].name)
self.assertEqual(2,
len(self.cat.get_workspaces(names=['topp', 'sde'])))
self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde')))
def testStores(self):
self.assertEqual(0, len(self.cat.get_stores("nonexistentstore")))
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(workspace=topp)))
self.assertEqual(2, len(self.cat.get_stores(workspace=sf)))
self.assertEqual(2, len(self.cat.get_stores(workspace='sf')))
self.assertEqual(
2, len(self.cat.get_stores(names='states_shapefile, sfdem')))
self.assertEqual(
2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem'])))
self.assertEqual("sfdem", self.cat.get_stores("sfdem")[-1].name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", "topp").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states",
self.cat.get_resource("states", states).name)
self.assertEqual("states",
self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title, states.abstract, states.native_bbox,
states.latlon_bbox, states.projection, states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem",
self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer":
continue
# looking for same name
ro = self.cat.get_resource(rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
# print "Updated layer: " + rd.name
count += 1
# print "Total updated layers: " + str(count)
def testLayers(self):
expected = set([
"Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies",
"tasmania_state_boundaries", "tasmania_cities", "states",
"poly_landmarks", "tiger_roads", "poi", "giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assertTrue(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assertEqual("states", states.name)
self.assertIsInstance(states.resource, ResourceInfo)
self.assertEqual(set(s.name for s in states.styles),
set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assertTrue(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assertEqual("tasmania", tas.name)
self.assertIsInstance(tas, LayerGroup)
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assertIsNotNone(self.cat.get_workspace("topp"))
tas2 = self.cat.create_layergroup("tasmania_reloaded",
tas.layers,
workspace="topp")
self.cat.save(tas2)
self.assertIsNone(self.cat.get_layergroup("tasmania_reloaded"))
self.assertIsNotNone(
self.cat.get_layergroup("tasmania_reloaded", "topp"))
tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp")
self.assertEqual("tasmania_reloaded", tas2.name)
self.assertIsInstance(tas2, LayerGroup)
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
self.assertEqual(tas2.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld",
self.cat.get_style("population").filename)
self.assertEqual("population",
self.cat.get_style("population").sld_name)
self.assertIsNone(self.cat.get_style('non-existing-style'))
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
self.assertEqual(
self.cat.get_store(workspace="best workspace ever",
name="best store ever"), None)
self.cat.get_layer("best layer ever")
self.cat.get_layergroup("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = [
'workspaces', 'test', 'datastores', u'operaci\xf3n_repo',
'featuretypes.xml'
]
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(
u, self.cat.service_url +
"/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = [
'workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml'
]
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(
u, self.cat.service_url +
"/workspaces/test/datastores/test-repo/featuretypes.xml")
def calccosts(cf, layer_name, json_matrix):
new_layer = "classroads_{}".format(int(1000000 * time.time()))
# calccost function calculates recalcutes the visualisation caterogies
engine = create_engine(
"postgresql+psycopg2://"
+ cf.get("PostGis", "user")
+ ":"
+ cf.get("PostGis", "pass")
+ "@"
+ cf.get("PostGis", "host")
+ ":"
+ str(cf.get("PostGis", "port"))
+ "/"
+ cf.get("PostGis", "db"),
strategy="threadlocal",
pool_pre_ping=True,
)
# try: # TODO make the ra2ce schema name as a parameter (input to the service)
strSql = """create table temp.{new_layer} as select geom, societal_class, repair_class, 0 as visclass from public.{layer_name}_priority;""".format(
new_layer=new_layer, layer_name=layer_name
)
ressql = engine.execute(strSql)
print("create temp table", ressql)
# except exc.DBAPIError as e:
# an exception is raised, Connection is invalidated.
# if e.connection_invalidated:
# logging.info("Connection was invalidated!")
data = json.loads(json_matrix)
values = data["values"]
for societalIndex in range(len(values)):
for repairIndex in range(len(values[societalIndex])):
strSql = """update temp.{new_layer} set visclass = {val}
where societal_class = {s} and repair_class = {r};""".format(
new_layer=new_layer,
val=values[societalIndex][repairIndex],
s=societalIndex + 1,
r=repairIndex + 1,
)
ressql = engine.execute(strSql)
ressql.close()
# publish the layer in geoserver
cat = Catalog(
"{}/rest/".format(cf.get("GeoServer", "host")),
username=cf.get("GeoServer", "user"),
password=cf.get("GeoServer", "pass"),
)
print("cat", cat)
datastore = cat.get_store("temp")
ft = cat.publish_featuretype(
new_layer, datastore, native_crs="EPSG:4326", jdbc_virtual_table=""
)
cat.save(ft)
layer = cat.get_layer(new_layer)
print("layer", layer)
# check if shapefile is line or point
# TODO check styles with Gerrit
style = "ra2ce"
# set style and save layer
# layer._set_default_style(style)
layer.default_style = style
cat.save(layer)
res = writeOutput(cf=cf, wmslayer="ra2ce-dr:{}".format(new_layer), defstyle=style)
# time.sleep(2)
return res
class GeoserverManager():
gs_master = None
gs_slaves = []
def __init__(self, config, disable_ssl_certificate_validation=False):
# settings
config = config["settings"]["geoserver"]
# log.info(config["geoserver_master"])
# log.info(config["password"])
# log.info(config["username"])
self.config = config
if "geoserver_master" in config:
print config["geoserver_master"]
self.gs_master = Catalog(config["geoserver_master"], self.config["username"], self.config["password"])
else:
raise Exception('config["geoserver_master"] has to be mapped to a running Geoserver')
if "geoserver_slaves" in config:
for gs_slave in config["geoserver_slaves"]:
self.gs_slaves.append(Catalog(gs_slave, self.config["username"], self.config["password"]))
def _initialize_geoserver(self, geoserver):
geoserver.username = self.config["username"]
geoserver.password = self.config["password"]
def publish_coveragestore(self, path, data, overwrite=False, reload_gs_slaves=True):
log.info(data)
workspace = self.gs_master.get_default_workspace() if "workspace" not in data else self.check_workspace(data["workspace"], False)
if "layerName" not in data:
raise Exception("No layerName found in the metadata")
name = data["layerName"]
# publish
self.gs_master.create_coveragestore(name, path, workspace, overwrite)
# setting default style
if "defaultStyle" in data:
self.set_style(name, workspace.name, data["defaultStyle"], False)
# reload geoserver slaves
if reload_gs_slaves:
self.reload_gs_slaves()
return True
def publish_postgis_table(self, data, reload_gs_slaves=True, native_crs=None, srs=None, overwrite=False):
try:
'''Publish a featuretype from data in an existing store'''
# @todo native_srs doesn't seem to get detected, even when in the DB
# metadata (at least for postgis in geometry_columns) and then there
# will be a misconfigured layer
name = data["layerName"]
title = data["title"] if "title" in data else name
store = self.gs_master.get_store(data["store"])
# #if native_crs is None: raise ValueError("must specify native_crs")
#srs = srs or native_crs
feature_type = FeatureType(self, store.workspace, store, name)
# because name is the in FeatureType base class, work around that
# and hack in these others that don't have xml properties
feature_type.dirty['name'] = name
# feature_type.dirty['srs'] = srs
# feature_type.dirty['nativeCRS'] = native_crs
feature_type.enabled = True
feature_type.title = title
headers = {
"Content-type": "application/xml",
"Accept": "application/xml"
}
headers, response = self.gs_master.http.request(store.resource_url, "POST", feature_type.message(), headers)
if headers.status != 201:
raise Exception(response, headers.status)
# defaultStyle
if "defaultStyle" in data:
# TODO: test it
self.set_style(name, store.workspace.name, data["defaultStyle"], False)
# reload geoservers
if reload_gs_slaves:
self.reload_gs_slaves()
return feature_type
except Exception, (response, status):
raise Exception(response, status)
def synchronizeLayers(self):
'''
Synchronizes layers/rules in the postgresql database, Geoserver, and Geofence. All three will match postgresql database.
'''
to_return = True
permanent_gs_layers = ['rapid_geoview_layers','rapid_feature','rapid_geoview']
rapid_layers = {lyr.uid for lyr in DataLayer.objects.all() if lyr.feature_set.count() > 0}
cat = Catalog(GEOSERVER_REST_ENDPOINT)
store = cat.get_store('RapidData')
gs_layers = {lyr.name for lyr in store.get_resources() if lyr.name not in permanent_gs_layers}
gf_rules_ids = {(rule.find('layer').text, int(rule.find('id').text)) for rule in ET.fromstring(getGeofenceRules('admin').content)[:-1]}
gf_rules = {rule[0] for rule in gf_rules_ids}
gs_layers_to_remove = gs_layers.difference(rapid_layers)
gs_layers_to_add = rapid_layers.difference(gs_layers)
gf_rules_to_remove = gf_rules.difference(rapid_layers)
gf_rules_to_add = rapid_layers.difference(gf_rules)
if len(gs_layers_to_add) == 0 and len(gs_layers_to_remove) == 0:
if len(gf_rules_to_add) == 0 and len(gf_rules_to_remove) == 0:
return to_return
elif len(gs_layers_to_remove) > 0:
for lyr_uid in gs_layers_to_remove:
print "Removing Geoserver layer {0}. . .".format(lyr_uid)
response = self.removeFeatureType(lyr_uid)
if response.status_code == 200:
print "Success!"
else:
print "Failed to remove layer {0}".format(lyr_uid)
to_return = False
print response.content
elif len(gs_layers_to_add) > 0:
for uid in gs_layers_to_add:
print "Adding FeatureType to Geoserver: {0}".format(uid)
xml = self.createFeatureTypeFromUid(uid)
response = self.sendFeatureType(xml)
if response.status_code == 200:
print "Success!"
else:
print "Failed to add FeatureType {0}".format(uid)
to_return = False
print response.content
if len(gf_rules_to_add) == 0 and len(gf_rules_to_remove) == 0:
return to_return
elif len(gf_rules_to_remove) > 0:
gf_ids_to_remove = [rule for rule in gf_rules_ids if rule[0] in gf_rules_to_remove]
for rule in gf_ids_to_remove:
print "Removing Geofence rule for layer {0}".format(rule[0])
response = removeGeofenceRule(rule[1])
if response.status_code == 200:
print "Success!"
else:
print "Failed to remove layer {0}".format(lyr_uid)
to_return = false
print response.content
elif len(gf_rules_to_add) > 0:
for rule in gf_rules_to_add:
response = addGeofenceRule(layer=rule)
if response.status_code == 200:
print "Success!"
else:
print "Failed to add Geofence rule for layer {0}".format(rule)
to_return = False
print response.content
return to_return
The GeoServer USERNAME and PASSWORD *are not* the ones configured on GeoNode.
It is mandatory to create a GeoServer manager user (by default admin/geoserver)
with WRITE (or ADMIN) rights and BASIC AUTH access enabled.
"""
url = "http://*****:*****@time [" + json.dumps(feature["properties"]["time"]) + "]"
granule_id = feature["id"]
gs_catalog.mosaic_delete_granule(coverages['coverages']['coverage'][0]['name'], store, granule_id)
print "Deleting file [" + json.dumps(feature["properties"]["location"]) + "]"
try:
os.remove(feature["properties"]["location"])
except Exception as e:
print "Could not delete the GeoTIFF Granule [" + str(feature["properties"]["location"]) + "]: " + str(e)
#!/usr/bin/env python
'''
gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API.
The project is distributed under a MIT License .
'''
__author__ = "David Winslow"
__copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans"
__license__ = "MIT"
from geoserver.catalog import Catalog
cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver")
ds = cat.create_datastore(name)
ds.connection_parameters.update(
host="localhost",
port="5432",
database="gis",
user="******",
passwd="",
dbtype="postgis")
cat.save(ds)
ds = cat.get_store(name)
components = \
dict((ext, "myfile." + ext) for ext in ["shp", "prj", "shx", "dbf"])
cat.add_data_to_store(ds, "mylayer", components)
def export(uid):
"""This function is used to show and handle the export options and uses the GeoServer REST API to export results.
This function uses the GeoServer configuration client library by boundlessgeo, see:
https://github.com/boundlessgeo/gsconfig
To export the results, the GeoServer application has to be running and should be correctly set up to have access
to the database and the styles should already be imported, see geoserver styles folder.
GET request: Renders and returns a site showing export options.
POST request: Gets the export options from the user and exports the results if they are defined in the options.
The export is made by using GeoServer configuration client library. After the export, the newly created layers
should be visible in the GeoServer web interface and the WM(T)S links should also work. These links can then be
used to display the WM(T)S layers in JOSM or a gis.
Remarks: This function will be redesigned in future versions
"""
uid = uid.encode("ISO-8859-1")
if request.method == "POST":
dm = DevMap.query.filter_by(uid=uid).first()
if dm.owner == current_user or dm.owner == User.query.filter_by(username="******").first():
title = request.form["title"]
listedmap = False
if "listed" in request.form:
listedmap = True
dm.title = title
dm.listed = listedmap
cat = Catalog(gs_url + "rest")
cat.username = gs_user
cat.password = gs_password
ws = None
try:
ws = cat.get_workspace(gs_workspace)
except socket.error, e:
db.session.add(dm)
db.session.commit()
return render_template("export.html", uid=uid, error=e, dm=dm)
st = cat.get_store(gs_store, ws)
if "maxdevgrid" in request.form:
feattype = "odf_" + uid + "_maxdevgrid"
if dm.wmsmaxdevgrid:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":maxdevgrid")
cat.save(l)
dm.wmsmaxdevgrid = True
else:
dm.wmsmaxdevgrid = True
if "posdevlines" in request.form:
feattype = "odf_" + uid + "_posdevlines"
if dm.wmsposdevlines:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":posdevlines")
cat.save(l)
dm.wmsposdevlines = True
else:
dm.wmsposdevlines = False
if "absdevgrid" in request.form:
feattype = "odf_" + uid + "_absdevgrid"
if dm.wmsabsdevgrid:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":absdevgrid")
cat.save(l)
dm.wmsabsdevgrid = True
else:
dm.wmsabsdevgrid = False
if "matchingrategrid" in request.form:
feattype = "odf_" + uid + "_matchingrategrid"
if dm.wmsmatchingrategrid:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":matchingrategrid")
cat.save(l)
dm.wmsmatchingrategrid = True
else:
dm.wmsmatchingrategrid = False
if "unmatchedref" in request.form:
feattype = "odf_" + uid + "_unmatchedref"
if dm.wmsunmatchedref:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":ReferenceLines")
cat.save(l)
dm.unmatchedref = True
else:
dm.unmatchedref = False
if "unmatchedosm" in request.form:
feattype = "odf_" + uid + "_unmatchedosm"
if dm.unmatchedosm:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":OSMLines")
cat.save(l)
dm.wmsunmatchedosm = True
else:
dm.wmsunmatchedosm = False
if "matchedref" in request.form:
feattype = "odf_" + uid + "_matchedref"
if dm.wmsmatchedref:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":ReferenceLines")
cat.save(l)
dm.wmsmatchedref = True
else:
dm.wmsmatchedref = False
if "matchedosm" in request.form:
feattype = "odf_" + uid + "_matchedosm"
if dm.wmsmatchedosm:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":OSMLines")
cat.save(l)
dm.wmsmatchedosm = True
else:
dm.wmsmatchedosm = False
if "minlevenshtein" in request.form:
feattype = "odf_" + uid + "_minlevenshtein"
if dm.wmsminlevenshtein:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":ReferenceLines")
cat.save(l)
dm.wmsminlevenshtein = True
else:
dm.wmsminlevenshtein = False
if "maxlevenshtein" in request.form:
feattype = "odf_" + uid + "_maxlevenshtein"
if dm.wmsmaxlevenshtein:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
cat.save(ft)
l = cat.get_layer(feattype)
l._set_default_style(gs_workspace + ":ReferenceLines")
cat.save(l)
dm.wmsmaxlevenshtein = True
else:
dm.wmsmaxlevenshtein = False
db.session.add(dm)
db.session.commit()
return render_template("finished.html", uid=uid)
else:
return render_template("export.html", uid=uid, error="No User", dm=dm)
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
self.gs_version = self.cat.get_short_version()
def testGSVersion(self):
version = self.cat.get_version()
pat = re.compile('\d\.\d+')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspaces(names="topp")[-1].name)
self.assertEqual(2, len(self.cat.get_workspaces(names=['topp', 'sde'])))
self.assertEqual(2, len(self.cat.get_workspaces(names='topp, sde')))
self.assertEqual("topp", self.cat.get_workspace("topp").name)
self.assertIsNone(self.cat.get_workspace("blahblah-"))
def testStores(self):
self.assertEqual(0, len(self.cat.get_stores(names="nonexistentstore")))
topp = self.cat.get_workspaces("topp")[0]
sf = self.cat.get_workspaces("sf")[0]
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(workspaces=topp)))
self.assertEqual(2, len(self.cat.get_stores(workspaces=sf)))
self.assertEqual(2, len(self.cat.get_stores(workspaces='sf')))
self.assertEqual(2, len(self.cat.get_stores(names='states_shapefile, sfdem')))
self.assertEqual(2, len(self.cat.get_stores(names=['states_shapefile', 'sfdem'])))
self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0].name)
self.assertEqual("states_shapefile", self.cat.get_stores(names="states_shapefile")[0].name)
self.assertEqual("sfdem", self.cat.get_stores(names="sfdem", workspaces=sf.name)[0].name)
self.assertEqual("sfdem", self.cat.get_stores(names="sfdem")[0].name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
self.assertIsNone(self.cat.get_store("blah+blah-"))
def testResources(self):
topp = self.cat.get_workspaces("topp")[0]
sf = self.cat.get_workspaces("sf")[0]
states = self.cat.get_stores(names="states_shapefile", workspaces=topp.name)[0]
sfdem = self.cat.get_stores(names="sfdem", workspaces=sf.name)[0]
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(2, len(self.cat.get_resources(stores=[states.name, sfdem.name])))
self.assertEqual(11, len(self.cat.get_resources(workspaces=[topp.name, sf.name])))
self.assertEqual("states", self.cat.get_resources(names="states", stores=states.name)[0].name)
self.assertEqual("states", self.cat.get_resources(names="states", workspaces=topp.name)[0].name)
self.assertEqual("states", self.cat.get_resources(names="states")[0].name)
self.assertEqual("states", self.cat.get_resource("states").name)
self.assertIsNone(self.cat.get_resource("blah+1blah-2"))
states = self.cat.get_resources(names="states")[0]
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", stores=sfdem.name)[0].name)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem", workspaces=sf.name)[0].name)
self.assertEqual("sfdem", self.cat.get_resources(names="sfdem")[0].name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer":
continue
# looking for same name
ro = self.cat.get_resources(names=rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
count += 1
def testLayers(self):
if self.gs_version >= "2.13":
expected = set([
'sf:roads',
'sf:sfdem',
'nurc:mosaic',
'tiger:giant_polygon',
'sf:bugsites',
'topp:states',
'sf:streams',
'tiger:poly_landmarks',
'tiger:poi',
'topp:tasmania_water_bodies',
'tiger:tiger_roads',
'topp:tasmania_roads',
'nurc:Pk50095',
'topp:tasmania_cities',
'nurc:Img_Sample',
'sf:restricted',
'nurc:Arc_Sample',
'sf:archsites',
'topp:tasmania_state_boundaries'
])
else:
expected = set([
"Arc_Sample",
"Pk50095",
"Img_Sample",
"mosaic",
"sfdem",
"bugsites",
"restricted",
"streams",
"archsites",
"roads",
"tasmania_roads",
"tasmania_water_bodies",
"tasmania_state_boundaries",
"tasmania_cities",
"states",
"poly_landmarks",
"tiger_roads",
"poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups(names=["tasmania", "tiger-ny", "spearfish"]))
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroups(names="tasmania")[0]
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
if self.gs_version >= "2.13":
self.assertEqual(tas.layers, [
'topp:tasmania_state_boundaries',
'topp:tasmania_water_bodies',
'topp:tasmania_roads',
'topp:tasmania_cities'
], tas.layers)
else:
self.assertEqual(tas.layers, [
'tasmania_state_boundaries',
'tasmania_water_bodies',
'tasmania_roads',
'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assert_(self.cat.get_workspaces("topp")[0] is not None)
tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp")
self.cat.save(tas2)
self.assertEqual(1, len(self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")))
tas2 = self.cat.get_layergroups(names='tasmania_reloaded', workspaces="topp")[0]
self.assert_("tasmania_reloaded", tas2.name)
self.assert_(isinstance(tas2, LayerGroup))
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
if self.gs_version >= "2.13":
self.assertEqual(tas2.layers, [
'topp:tasmania_state_boundaries',
'topp:tasmania_water_bodies',
'topp:tasmania_roads',
'topp:tasmania_cities'
], tas2.layers)
else:
self.assertEqual(tas2.layers, [
'tasmania_state_boundaries',
'tasmania_water_bodies',
'tasmania_roads',
'tasmania_cities'
], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_styles("population")[0].name)
self.assertEqual("popshade.sld", self.cat.get_styles("population")[0].filename)
self.assertEqual("population", self.cat.get_styles("population")[0].sld_name)
self.assertEqual("population", self.cat.get_style("population").sld_name)
self.assertIsNone(self.cat.get_style("blah+#5blah-"))
self.assertEqual(0, len(self.cat.get_styles('non-existing-style')))
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_styles("best style ever")
self.cat.get_workspaces("best workspace ever")
self.assertEqual(0, len(self.cat.get_stores(workspaces="best workspace ever", names="best store ever")))
self.cat.get_layer("best layer ever")
self.cat.get_layergroups("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = build_url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
def delete(uid):
"""A function to delete a deviation map, all its tables, files and GeoServer layers.
GET request: Renders and returns a site showing delete options.
POST request: Gets delete options chosen by user and uses them to delete the chosen parts of the deviation map.
To delete GeoServer layers the GeoServer configuration client library is used.
"""
uid = uid.encode("ISO-8859-1")
if request.method == "POST":
dm = DevMap.query.filter_by(uid=uid).first()
if (
current_user.is_authenticated()
and dm.owner == current_user
or dm.owner == User.query.filter_by(username="******").first()
):
if (
dm.wmsposdevlines
or dm.wmsmaxdevgrid
or dm.wmsabsdevgrid
or dm.wmsmatchingrategrid
or dm.wmsunmatchedref
or dm.wmsunmatchedosm
or dm.wmsmatchedref
or dm.wmsmatchedosm
or dm.wmsminlevenshtein
or dm.wmsmaxlevenshtein
):
cat = Catalog(gs_url + "rest")
cat.username = gs_user
cat.password = gs_password
ws = None
try:
ws = cat.get_workspace(gs_workspace)
except socket.error, e:
detail = "GeoServer is not available. Make sure that it is running and the connection is ok."
return render_template("error.html", err=e, detail=detail)
st = cat.get_store(gs_store, ws)
if "deletemaxdevgrid" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_maxdevgrid"
if dm.wmsmaxdevgrid:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsmaxdevgrid = False
if "deleteposdevlines" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_posdevlines"
if dm.wmsposdevlines:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsposdevlines = False
if "deleteabsdevgrid" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_absdevgrid"
if dm.wmsabsdevgrid:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
ft = cat.publish_featuretype(feattype, st, "EPSG:4326")
if ft is not None:
cat.delete(ft)
dm.wmsabsdevgrid = False
if "deletematchingrategrid" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_matchingrategrid"
if dm.wmsmatchingrategrid:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsmatchingrategrid = False
if "deleteunmatchedref" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_unmatchedref"
if dm.wmsunmatchedref:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.deleteunmatchedref = False
if "deleteunmatchedosm" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_unmatchedosm"
if dm.wmsunmatchedosm:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsunmatchedosm = False
if "deletematchedref" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_matchedref"
if dm.wmsmatchedref:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsmatchedref = False
if "deletematchedosm" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_matchedosm"
if dm.wmsmatchedosm:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsmatchedosm = False
if "deleteminlevenshtein" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_minlevenshtein"
if dm.wmsminlevenshtein:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsminlevenshtein = False
if "deletemaxlevenshtein" in request.form or "deleteall" in request.form:
feattype = "odf_" + uid + "_maxlevenshtein"
if dm.wmsmaxlevenshtein:
l = cat.get_layer(feattype)
if l is not None:
cat.delete(l)
dm.wmsmaxlevenshtein = False
if "deleteall" in request.form:
folder = secure_filename(uid)
folder = os.path.join(app.config["UPLOAD_FOLDER"], folder)
shutil.rmtree(folder, True)
db.engine.execute("drop table if exists odf_" + uid + "_ref")
db.engine.execute("drop table if exists odf_" + uid + "_ref_presplitted")
db.engine.execute("drop table if exists odf_" + uid + "_ref_splitted")
db.engine.execute("drop table if exists odf_" + uid + "_found")
db.engine.execute("drop table if exists odf_" + uid + "_ref_junctions")
db.engine.execute("drop table if exists odf_" + uid + "_ref_points")
db.engine.execute("drop table if exists odf_" + uid + "_ref_cutpoints")
db.engine.execute("drop table if exists odf_" + uid + "_ref_cutcheckpoints")
db.engine.execute("drop table if exists odf_" + uid + "_osm")
db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted")
db.engine.execute("drop table if exists odf_" + uid + "_osm_splitted")
db.engine.execute("drop table if exists odf_" + uid + "_osm_junctions")
db.engine.execute("drop table if exists odf_" + uid + "_osm_points")
db.engine.execute("drop table if exists odf_" + uid + "_osm_cutpoints")
db.engine.execute("drop table if exists odf_" + uid + "_osm_cutcheckpoints")
db.engine.execute("drop table if exists odf_" + uid + "_unmatchedref;")
db.engine.execute("drop table if exists odf_" + uid + "_unmatchedosm;")
db.engine.execute("drop table if exists odf_" + uid + "_minlevenshtein;")
db.engine.execute("drop table if exists odf_" + uid + "_maxlevenshtein;")
db.engine.execute("drop table if exists odf_" + uid + "_grid;")
db.engine.execute("drop table if exists odf_" + uid + "_maxdevgrid;")
db.engine.execute("drop table if exists odf_" + uid + "_matchingrategrid;")
db.engine.execute("drop table if exists odf_" + uid + "_deviationlines")
db.engine.execute("drop table if exists odf_" + uid + "_junction_deviationlines")
if DEBUG:
db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_cutcheckpoints")
db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_cutpoints")
db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_junctions")
db.engine.execute("drop table if exists odf_" + uid + "_osm_presplitted_points")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_cutcheckpoints")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_cutpoints")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_junction_devvec")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_junctions")
db.engine.execute("drop table if exists odf_" + uid + "_ref_corrected_presplitted_points")
db.engine.execute("drop table if exists odf_" + uid + "_result")
if "deleteall" not in request.form:
db.session.add(dm)
db.session.commit()
return render_template("delete.html", uid=uid, dm=dm, error=None)
else:
db.session.delete(dm)
db.session.commit()
return redirect(url_for("basic.index"))
else:
return render_template("error.html", err="You are not allowed to delete this map!")
class GeoGigUploaderBase(ImportHelper):
def __init__(self, *args, **kwargs):
super(GeoGigUploaderBase, self).__init__(*args, **kwargs)
setUpModule() # this isn't available when being used in other module
def setUp(self):
self.admin_user = self.create_user('admin', 'admin', is_superuser=True)
self.non_admin_user = self.create_user('non_admin', 'non_admin')
self.catalog = Catalog(
ogc_server_settings.internal_rest,
*ogc_server_settings.credentials
)
if self.catalog.get_workspace('geonode') is None:
self.catalog.create_workspace('geonode', 'http://www.geonode.org/')
self.workspace = 'geonode'
self.datastoreNames = []
def tearDown(self):
"""Clean up geoserver/geogig catalog.
"""
# delete stores (will cascade to delete layers)
for store_name in self.datastoreNames:
self.catalog.delete(
self.catalog.get_store(store_name), recurse=True)
# delete repository reference in geoserver
for store_name in self.datastoreNames:
self.remove_geogig_repo(store_name)
# geoserver can leave connections open - HACK HACK HACK
self.free_geogig_connections()
# HACK HACK HACK -- sometimes connections from geoserver to geogig are left open. This kills the postgresql backend!
# this is a major hammer. Once geoserver/geogig are better at cleaning up, remove this.
def free_geogig_connections(self):
with db.connections["geogig"].cursor() as c:
c.execute(
"select pg_terminate_backend(pid) from pg_stat_activity where application_name = 'PostgreSQL JDBC Driver' or application_name='geogig'")
# aggressive delete of the repo (mostly cleans up after itself)
# call the geogig rest API DELETE
def remove_geogig_repo(self, ref_name):
username = ogc_server_settings.credentials.username
password = ogc_server_settings.credentials.password
url = ogc_server_settings.rest
http = httplib2.Http(disable_ssl_certificate_validation=False)
http.add_credentials(username, password)
netloc = urlparse(url).netloc
http.authorizations.append(
httplib2.BasicAuthentication(
(username, password),
netloc,
url,
{},
None,
None,
http
))
rest_url = ogc_server_settings.LOCATION + \
"geogig/repos/" + ref_name + "/delete.json"
resp, content = http.request(rest_url, 'GET')
response = json.loads(content)
token = response["response"]["token"]
rest_url = ogc_server_settings.LOCATION + \
"geogig/repos/" + ref_name + "?token=" + token
resp, content = http.request(rest_url, 'DELETE')
# convenience method to load in the test dataset
# return a (geonode) layer
# the layer will be in Geoserver and Geonode
# self.catalog.get_layer(layer.name) -- to get the Geoserver Layer
def fully_import_file(self, path, fname, start_time_column, end_time_column=None):
# setup time
if end_time_column is None:
time_config = {'convert_to_date': [start_time_column], 'start_date': start_time_column,
'configureTime': True}
else:
time_config = {'convert_to_date': [start_time_column, end_time_column], 'start_date': start_time_column,
'end_date': end_time_column, 'configureTime': True}
name = os.path.splitext(fname)[0] + "_" + str(uuid.uuid1())[:8]
self.datastoreNames.append(name) # remember for future deletion
full_fname = os.path.join(path, fname)
configs = self.prepare_file_for_import(full_fname)
configs[0].update({'name': name})
configs[0].update({'layer_name': name})
configs[0].update(time_config)
# configure the datastore/repo
configs[0]['geoserver_store'] = {}
configs[0]['geoserver_store']['type'] = 'geogig'
configs[0]['geoserver_store']['name'] = name
configs[0]['geoserver_store']['create'] = 'true'
configs[0]['geoserver_store']['branch'] = 'master'
configs[0]['geoserver_store']['geogig_repository'] = "geoserver://" + name
result = self.generic_import(fname, path=path, configs=configs)
return result
def import_file(self, path, configs=None):
"""Imports the file.
"""
if configs is None:
configs = []
self.assertTrue(os.path.exists(path), path)
# run ogr2ogr
ogr = OGRImport(path)
layers = ogr.handle(configuration_options=configs)
return layers
def generic_import(self, filename, path, configs=None):
if configs is None:
configs = [{'index': 0}]
path = os.path.join(path, filename)
results = self.import_file(path, configs=configs)
layer_results = []
for result in results:
layer = Layer.objects.get(name=result[0])
self.assertEqual(layer.srid, 'EPSG:4326')
self.assertTrue(layer.store in self.datastoreNames)
self.assertEqual(layer.storeType, 'dataStore')
if not path.endswith('zip'):
self.assertGreaterEqual(
layer.attributes.count(),
DataSource(path)[0].num_fields
)
layer_results.append(layer)
return layer_results[0]
def prepare_file_for_import(self, filepath):
""" Prepares the file path provided for import; performs some housekeeping, uploads & configures the file.
Returns a list of dicts of the form {'index': <layer_index>, 'upload_layer_id': <upload_layer_id>}
these may be used as configuration options for importing all of the layers in the file.
"""
# Make a copy of the test file, as it's removed in configure_upload()
filename = os.path.basename(filepath)
tmppath = os.path.join('/tmp', filename)
shutil.copy(filepath, tmppath)
# upload & configure_upload expect closed file objects
# This is heritage from originally being closely tied to a view passing request.Files
of = open(tmppath, 'rb')
of.close()
files = [of]
uploaded_data = self.upload(files, self.admin_user)
self.configure_upload(uploaded_data, files)
configs = [{'index': l.index, 'upload_layer_id': l.id}
for l in uploaded_data.uploadlayer_set.all()]
return configs
def create_user(self, username, password, **kwargs):
"""Convenience method for creating users.
"""
user, created = User.objects.get_or_create(username=username, **kwargs)
if created:
user.set_password(password)
user.save()
return user
def _register_cascaded_service(url, type, name, username, password, wms=None, owner=None, parent=None):
"""
Register a service as cascading WMS
"""
try:
service = Service.objects.get(base_url=url)
return_dict = {}
return_dict["service_id"] = service.pk
return_dict["msg"] = "This is an existing Service"
return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200)
except:
# TODO: Handle this error properly
pass
if wms is None:
wms = WebMapService(url)
# TODO: Make sure we are parsing all service level metadata
# TODO: Handle for setting ServiceProfiletRole
service = Service.objects.create(
base_url=url,
type=type,
method="C",
name=name,
version=wms.identification.version,
title=wms.identification.title,
abstract=wms.identification.abstract,
online_resource=wms.provider.url,
owner=owner,
parent=parent,
)
service.keywords = ",".join(wms.identification.keywords)
service.save()
service.set_default_permissions()
if type in ["WMS", "OWS"]:
# Register the Service with GeoServer to be cascaded
cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password)
cascade_ws = cat.get_workspace(name)
if cascade_ws is None:
cascade_ws = cat.create_workspace(name, "http://geonode.org/" + name)
# TODO: Make sure there isn't an existing store with that name, and
# deal with it if there is
try:
cascade_store = cat.get_store(name, cascade_ws)
except:
cascade_store = cat.create_wmsstore(name, cascade_ws, username, password)
cascade_store.capabilitiesURL = url
cascade_store.type = "WMS"
cat.save(cascade_store)
available_resources = cascade_store.get_resources(available=True)
elif type == "WFS":
# Register the Service with GeoServer to be cascaded
cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password)
# Can we always assume that it is geonode?
cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE)
if cascade_ws is None:
cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, "http://geonode.org/cascade")
try:
wfs_ds = cat.get_store(name, cascade_ws)
except:
wfs_ds = cat.create_datastore(name, cascade_ws)
connection_params = {
"WFSDataStoreFactory:MAXFEATURES": "0",
"WFSDataStoreFactory:TRY_GZIP": "true",
"WFSDataStoreFactory:PROTOCOL": "false",
"WFSDataStoreFactory:LENIENT": "true",
"WFSDataStoreFactory:TIMEOUT": "3000",
"WFSDataStoreFactory:BUFFER_SIZE": "10",
"WFSDataStoreFactory:ENCODING": "UTF-8",
"WFSDataStoreFactory:WFS_STRATEGY": "nonstrict",
"WFSDataStoreFactory:GET_CAPABILITIES_URL": url,
}
if username and password:
connection_params["WFSDataStoreFactory:USERNAME"] = username
connection_params["WFSDataStoreFactory:PASSWORD"] = password
wfs_ds.connection_parameters = connection_params
cat.save(wfs_ds)
available_resources = wfs_ds.get_resources(available=True)
# Save the Service record
service, created = Service.objects.get_or_create(type=type, method="C", base_url=url, name=name, owner=owner)
service.save()
service.set_default_permissions()
elif type == "WCS":
return HttpResponse("Not Implemented (Yet)", status=501)
else:
return HttpResponse(
"Invalid Method / Type combo: " + "Only Cascaded WMS, WFS and WCS supported",
mimetype="text/plain",
status=400,
)
message = "Service %s registered" % service.name
return_dict = [
{
"status": "ok",
"msg": message,
"service_id": service.pk,
"service_name": service.name,
"service_title": service.title,
"available_layers": available_resources,
}
]
if settings.USE_QUEUE:
# Create a layer import job
WebServiceHarvestLayersJob.objects.get_or_create(service=service)
else:
_register_cascaded_layers(service)
return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200)
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(
host="localhost", port="5432", database="db", user="******",
passwd="password", dbtype="postgis")
self.cat.save(ds)
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(
host="localhost", port="5432", database="db", user="******",
passwd="password", dbtype="postgis")
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
def testCoverageStoreCreate(self):
ds = self.cat.create_coveragestore2("coverage_gsconfig")
ds.data_url = "file:data/mytiff.tiff"
self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats))
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
# TODO: Uploading WorldImage file no longer works???
# ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
# self.assertRaises(
# ConflictingDataError,
# lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# )
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None, skip_unadvertised=False):
"""Configure the layers available in GeoServer in GeoNode.
It returns a list of dictionaries with the name of the layer,
the result of the operation and the errors and traceback if it failed.
"""
# avoid circular import problem
from geonode.layers.models import set_attributes
if console is None:
console = open(os.devnull, 'w')
if verbosity > 1:
print >> console, "Inspecting the available layers in GeoServer ..."
cat = Catalog(ogc_server_settings.internal_rest, _user, _password)
if workspace is not None:
workspace = cat.get_workspace(workspace)
resources = cat.get_resources(workspace=workspace)
elif store is not None:
store = cat.get_store(store)
resources = cat.get_resources(store=store)
else:
resources = cat.get_resources(workspace=workspace)
if filter:
resources = [k for k in resources if filter in k.name]
# filter out layers depending on enabled, advertised status:
resources = [k for k in resources if k.enabled == "true"]
if skip_unadvertised: resources = [k for k in resources if k.advertised == "true" or k.advertised == None]
# TODO: Should we do something with these?
# i.e. look for matching layers in GeoNode and also disable?
disabled_resources = [k for k in resources if k.enabled == "false"]
number = len(resources)
if verbosity > 1:
msg = "Found %d layers, starting processing" % number
print >> console, msg
output = {
'stats': {
'failed':0,
'updated':0,
'created':0,
},
'layers': []
}
start = datetime.datetime.now()
for i, resource in enumerate(resources):
name = resource.name
store = resource.store
workspace = store.workspace
try:
# Avoid circular import problem
from geonode.layers.models import Layer
layer, created = Layer.objects.get_or_create(name=name, defaults = {
"workspace": workspace.name,
"store": store.name,
"storeType": store.resource_type,
"typename": "%s:%s" % (workspace.name.encode('utf-8'), resource.name.encode('utf-8')),
"title": resource.title or 'No title provided',
"abstract": resource.abstract or 'No abstract provided',
"owner": owner,
"uuid": str(uuid.uuid4())
})
layer.save()
# recalculate the layer statistics
set_attributes(layer, overwrite=True)
except Exception, e:
if ignore_errors:
status = 'failed'
exception_type, error, traceback = sys.exc_info()
else:
if verbosity > 0:
msg = "Stopping process because --ignore-errors was not set and an error was found."
print >> sys.stderr, msg
raise Exception('Failed to process %s' % resource.name.encode('utf-8'), e), None, sys.exc_info()[2]
else:
if created:
layer.set_default_permissions()
status = 'created'
output['stats']['created']+=1
else:
status = 'updated'
output['stats']['updated']+=1
msg = "[%s] Layer %s (%d/%d)" % (status, name, i+1, number)
info = {'name': name, 'status': status}
if status == 'failed':
output['stats']['failed']+=1
info['traceback'] = traceback
info['exception_type'] = exception_type
info['error'] = error
output['layers'].append(info)
if verbosity > 0:
print >> console, msg
def get_selected_raster(region, variable, date):
try:
conn = psycopg2.connect(
"dbname={0} user={1} host={2} password={3}".format(
cfg.connection['dbname'], cfg.connection['user'],
cfg.connection['host'], cfg.connection['password']))
cur = conn.cursor()
storename = region + '_' + variable + '_' + date
cat = Catalog(cfg.geoserver['rest_url'],
username=cfg.geoserver['user'],
password=cfg.geoserver['password'])
try:
something = cat.get_store(storename, cfg.geoserver['workspace'])
if not something:
print "No store"
else:
mean, stddev, min, max = get_vic_summary(
region, variable, date)
return storename, mean, stddev, min, max
except geoserver.catalog.FailedRequestError as e:
try:
sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format(
region, variable, date)
cur.execute(sql)
data = cur.fetchall()
mean, stddev, min, max = get_vic_summary(
region, variable, date)
rest_url = cfg.geoserver['rest_url']
if rest_url[-1] != "/":
rest_url = rest_url + '/'
headers = {
'Content-type': 'image/tiff',
}
request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format(
rest_url, cfg.geoserver['workspace'],
storename) # Creating the rest url
user = cfg.geoserver['user']
password = cfg.geoserver['password']
requests.put(
request_url,
headers=headers,
data=data[0][0],
auth=(user,
password)) # Creating the resource on the geoserver
conn.close()
return storename, mean, stddev, min, max
except Exception as e:
print e
return e
# if storename in geoserver_engine.list_stores()['result']:
# mean, stddev, min, max = get_vic_summary(region,variable,date)
# return storename,mean, stddev, min, max
# else:
# sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format(region,variable,date)
# cur.execute(sql)
# data = cur.fetchall()
#
# mean, stddev, min, max = get_vic_summary(region,variable,date)
#
# rest_url = cfg.geoserver['rest_url']
#
# if rest_url[-1] != "/":
# rest_url = rest_url + '/'
#
#
# headers = {
# 'Content-type': 'image/tiff',
# }
# request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format(rest_url,cfg.geoserver['workspace'] ,storename) # Creating the rest url
#
# user = cfg.geoserver['user']
# password = cfg.geoserver['password']
# requests.put(request_url, headers=headers, data=data[0][0],
# auth=(user,password)) # Creating the resource on the geoserver
#
# conn.close()
# return storename,mean,stddev,min,max
except Exception as e:
print e
return e
class GsConn:
def __init__(self, host, login, password, debug=False):
"""
Geoserver connection
"""
self.host = host
self.login = login
self.password = password
self.debug = debug
# Connect to server
self.cat = Catalog("http://%s/geoserver/rest" % host, login, password)
if self.debug is True:
print "Connected to geoserver"
def crate_workspace(self, name, overwrite=False):
"""
Creates a workspace
:param name: Workspace name.
:param overwrite: If True, delete existing workspace.
:return: None
"""
workspaces = [workspace.name for workspace in self.cat.get_workspaces()]
if name in workspaces and overwrite is True:
# ws2del = self.cat.get_workspace(name)
# self.cat.delete(ws2del, purge=True, recurse=True)
return None # NOTE: If we delete the workspace then all associated layers are lost.
elif name in workspaces and overwrite is False:
print "ERROR: Workspace %s already exists (use overwrite=True)." % name
self.cat.create_workspace(name, "http://%s/%s" % (self.host, name))
if self.debug is True:
print "Workspace %s available." % name
ws = self.cat.get_workspace(name)
ws.enabled = True
def create_pg_store(self, name, workspace, host, port, login, password, dbname, schema, overwrite=False):
"""
Creates datastore.
:param name: Name of the datastore.
:param workspace: Name of the workspace to use.
:param overwrite: If True replace datastore.
:return: None
"""
stores = [store.name for store in self.cat.get_stores()]
if name in stores and overwrite is True:
# st2del = self.cat.get_store(name)
# self.cat.delete(st2del, purge=True, recurse=True)
# self.cat.reload()
return None # NOTE: If we delete store, every layers associated with are lost.
elif name in stores and overwrite is False:
print "ERROR: Store %s already exists (use overwrite=True)." % name
ds = self.cat.create_datastore(name, workspace)
ds.connection_parameters.update(
host=host, port=port, user=login, passwd=password, dbtype="postgis", database=dbname, schema=schema
)
self.cat.save(ds)
ds = self.cat.get_store(name)
if ds.enabled is False:
print "ERROR: Geoserver store %s not enabled" % name
if self.debug is True:
print "Datastore %s created." % name
def publish_pg_layer(self, layer_table, layer_name, store, srid, overwrite=True):
"""
"""
existing_lyr = self.cat.get_layer("ma_carte:%s" % layer_table)
if existing_lyr is not None:
print "Layer ma_carte:%s already exists, deleting it." % layer_table
self.cat.delete(existing_lyr)
self.cat.reload()
ds = self.cat.get_store(store)
ft = self.cat.publish_featuretype(layer_table, ds, "EPSG:%s" % srid, srs="EPSG:4326")
ft.projection_policy = "REPROJECT_TO_DECLARED"
ft.title = layer_name
self.cat.save(ft)
if ft.enabled is False:
print "ERROR: Layer %s %s %s is not enabled."(ft.workspace.name, ft.store.name, ft.title)
if self.debug is True:
print "Layer %s>%s>%s published." % (ft.workspace.name, ft.store.name, ft.title)
def create_style_from_sld(self, style_name, sld_file, workspace, overwrite=True):
"""
"""
if self.cat.get_style(style_name) is not None:
print "Style %s already exists, deleting it." % style_name
style2del = self.cat.get_style(style_name)
self.cat.delete(style2del)
self.cat.create_style(
style_name, open(sld_file).read(), overwrite=overwrite
) # FIXME: if ", workspace=workspace" specified can't delete style
if self.debug is True:
print "Style %s created in Geoserver" % style_name
def apply_style_to_layer(self, layer_name, style_name):
"""
Apply a geoserver styler to a layer
"""
gs_layer = self.cat.get_layer(layer_name)
gs_style = self.cat.get_style(style_name)
# FIXME: Which works better?
# gs_layer.default_style = gs_style / gs_layer._set_default_style(gs_style)
# FIXME: Maybe indicate workspace when saving style then name the style as "workspace:style"
gs_layer._set_default_style(gs_style)
self.cat.save(gs_layer)
if self.debug is True:
print "Style applied to %s" % layer_name
cat.save(that_layer)
ooitest = cat.get_workspace("ooi_test")
data = cat.get_resources(store="asd",workspace="geonode")
print data
layer = cat.get_layer("1k time varying test")
print layer.href
cat.add_data_to_store
print layer
#drop those things not of interest
wmstest = cat.get_workspace("geonode")
try:
wmsstore = cat.get_store("ooi")
cat.delete(wmsstore)
except:
print "opps"
#create the data store
wmsstore = cat.create_datastore("ooi", wmstest)
wmsstore.capabilitiesURL = "http://www.geonode.org/"
wmsstore.type = "PostGIS"
#connection info
params = {
'Connection timeout': '20',
'Estimated extends': 'true',
'Expose primary keys': 'false',
def gs_slurp(ignore_errors=True,
verbosity=1,
console=None,
owner=None,
workspace=None,
store=None,
filter=None):
"""Configure the layers available in GeoServer in GeoNode.
It returns a list of dictionaries with the name of the layer,
the result of the operation and the errors and traceback if it failed.
"""
if console is None:
console = open(os.devnull, 'w')
if verbosity > 1:
print >> console, "Inspecting the available layers in GeoServer ..."
url = "%srest" % settings.OGC_SERVER['default']['LOCATION']
cat = Catalog(url, _user, _password)
if workspace is not None:
workspace = cat.get_workspace(workspace)
resources = cat.get_resources(workspace=workspace)
elif store is not None:
store = cat.get_store(store)
resources = cat.get_resources(store=store)
else:
resources = cat.get_resources(workspace=workspace)
if filter:
resources = [k for k in resources if filter in k.name]
# filter out layers explicitly disabled by geoserver
resources = [k for k in resources if k.enabled == "true"]
# TODO: Should we do something with these?
# i.e. look for matching layers in GeoNode and also disable?
disabled_resources = [k for k in resources if k.enabled == "false"]
number = len(resources)
if verbosity > 1:
msg = "Found %d layers, starting processing" % number
print >> console, msg
output = []
for i, resource in enumerate(resources):
name = resource.name
store = resource.store
workspace = store.workspace
try:
# Avoid circular import problem
from geonode.layers.models import Layer
layer, created = Layer.objects.get_or_create(
name=name,
defaults={
"workspace":
workspace.name,
"store":
store.name,
"storeType":
store.resource_type,
"typename":
"%s:%s" % (workspace.name.encode('utf-8'),
resource.name.encode('utf-8')),
"title":
resource.title or 'No title provided',
"abstract":
resource.abstract or 'No abstract provided',
"owner":
owner,
"uuid":
str(uuid.uuid4())
})
layer.save()
except Exception, e:
if ignore_errors:
status = 'failed'
exception_type, error, traceback = sys.exc_info()
else:
if verbosity > 0:
msg = "Stopping process because --ignore-errors was not set and an error was found."
print >> sys.stderr, msg
raise Exception(
'Failed to process %s' % resource.name.encode('utf-8'),
e), None, sys.exc_info()[2]
else:
if created:
layer.set_default_permissions()
status = 'created'
else:
status = 'updated'
msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number)
info = {'name': name, 'status': status}
if status == 'failed':
info['traceback'] = traceback
info['exception_type'] = exception_type
info['error'] = error
output.append(info)
if verbosity > 0:
print >> console, msg
class GsConfigTestCase(unittest.TestCase):
"""Test of the file manager"""
gs = None
workdir = None
def setUp(self):
cfg = ConfigParser.SafeConfigParser()
cfg.read((os.path.join(TEST_DIR,"tests.cfg")))
self.gs = GsConfig(cfg)
self.workdir = os.path.abspath(os.path.join(TEST_DIR,"workdir","data"))
gsUrl = self.gs.config.get("GeoServer", "url")
gsUser = self.gs.config.get("GeoServer", "user")
gsPassword = self.gs.config.get("GeoServer", "password")
self.direct_gs = Catalog(gsUrl, gsUser, gsPassword)
def tearDown(self):
class Req:
href=self.gs.config.get("GeoServer", "url") + "/styles/tmp_polygon"
if self.direct_gs.get_style("tmp_polygon"):
self.direct_gs.delete(Req())
### LAYERS ###
def test_getLayers(self):
layers = self.gs.getLayers()
print layers
# TODO: Add test
def test_getLayer(self):
layers = self.gs.getLayers()
layer = self.gs.getLayer(layers[0].name)
print layer
# TODO: Add test
### FEATURE STORE ###
def test_createFeatrueStore(self):
# TODO: make sure that source files exist
# check / delete layer
line_layer = self.direct_gs.get_layer("line_crs")
if line_layer is not None:
self.direct_gs.delete(line_layer)
# check / delete feature type
#
# sorry - it is not possible to delete feature type with gsconfig.py
#
# example in test
# https://github.com/dwins/gsconfig.py/blob/master/test/catalogtests.py#L424
# only shows how to disable it
#
# FOR NOW, PLEASE REMOVE THE FEATURE TYPE "line_crs0" MANUALLY
#
#line_feature = self.direct_gs.get_featuretype("line_crs0") # this function does not exist
#if line_feature is not None:
# self.direct_gs.delete(line_feature)
# TODO: remove the feature type using the gs rest api itself
# check / delete data store
# line_store = self.direct_gs.get_store("line_crs")
# if line_store is not None:
# self.direct_gs.delete(line_store)
self.assertEquals( True, None == self.direct_gs.get_layer("line_crs"), "The layer line_crs already exists. Please, remove it manually." )
self.assertEquals( True, None == self.direct_gs.get_store("line_crs"), "The store line_crs already exists. Please, remove it manually." )
self.gs.createFeatureStore(self.workdir+"/line_crs","TestWS","line_crs")
# Another problem - gs.createFeatureStore DOES NOT return the resource URI
# - hence WE DON'T know the name of the created layer!
# TODO: use the gs rest api itself and find out the name of the created layer
# - whether it is line_crs0, line_crs1 or what...
self.assertEquals( False, None == self.direct_gs.get_layer("line_crs"), "The layer line_crs is not there. Was it created under another name?" )
### STYLES ###
def test_getStyle(self):
"""Test get style"""
style = self.gs.getStyle("line")
self.assertTrue(style.find("<sld:Name>line</sld:Name>")>0)
def test_postStyle(self):
"""Test get files function"""
global style
self.gs.postStyle("tmp_polygon",style)
# TODO: add test
# FIXME: UploadError: SLD file styles/tmp_polygon.sld.sld already exists.
def test_putStyle(self):
"""Test get files function"""
global style
self.gs.postStyle("tmp_polygon",style)
self.gs.putStyle("tmp_polygon",style)
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testAbout(self):
about_html = self.cat.about()
self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld", self.cat.get_style("population").filename)
self.assertEqual("population", self.cat.get_style("population").sld_name)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.assertRaises(FailedRequestError,
lambda: self.cat.get_resource(
workspace="best workspace ever", store="best store ever",
name="best resource ever"))
self.cat.get_layer("best layer ever")
self.cat.get_layergroup("best layergroup ever")
def testUnicodeUrl(self):
"""
Tests that the geoserver.support.url function support unicode strings.
"""
# Test the url function with unicode
seg = ['workspaces', 'test', 'datastores', u'operaci\xf3n_repo', 'featuretypes.xml']
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/operaci%C3%B3n_repo/featuretypes.xml")
# Test the url function with normal string
seg = ['workspaces', 'test', 'datastores', 'test-repo', 'featuretypes.xml']
u = url(base=self.cat.service_url, seg=seg)
self.assertEqual(u, self.cat.service_url + "/workspaces/test/datastores/test-repo/featuretypes.xml")
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
class GeoServer(Server):
"""
Represents a server running GeoServer [1], an application that provides access to layers.
This class provides a concrete implementation of the more generic Server component (which is intentionally generic).
Currently the Server class does not dictate an interface for accessing resources but this class aims to present
GeoServer specific components (such as workspaces) as generic components (such as namespaces).
GeoServer instances typically represent individual instances (i.e. hosts are servers) rather than a wider and more
abstract platform offered by a service provider.
Information on layers and other resources are fetched using a combination of the GeoServer specific administrative
API [2] accessed through geoserver-restconfig [3] and OGC services accessed through OWSLib [4] (and currently
limited to WMS and WFS).
[1] https://geoserver.readthedocs.io/en/latest/
[2] https://geoserver.readthedocs.io/en/latest/rest/index.html
[3] https://pypi.org/project/geoserver-restconfig
[4] https://pypi.org/project/OWSLib/
"""
def __init__(
self,
server_id: str,
label: str,
hostname: str,
port: str,
api_path: str,
wms_path: str,
wfs_path: str,
username: str,
password: str,
):
"""
:param server_id: unique identifier, typically a ULID (Universally Unique Lexicographically Sortable Identifier)
:param label: a human readable, well-known, identifier for the server - typically based on the hostname
:param hostname: servers fully qualified hostname
:param port: port on which GeoServer is running (usually '80' or '8080')
:param api_path: URL path, relative to the root of the server, to the GeoServer API (usually '/geoserver/rest')
:param wms_path: URL path, relative to the root of the server, to the GeoServer WMS endpoint (usually
'/geoserver/ows?service=wms&version=1.3.0&request=GetCapabilities')
:param wfs_path: URL path, relative to the root of the server, to the GeoServer WFS endpoint (usually
'/geoserver/ows?service=wfs&version=2.0.0&request=GetCapabilities')
:param username: username for account to use for GeoServer API
:param password: password for account to use for GeoServer API
"""
endpoint = build_base_data_source_endpoint(data_source={
"hostname": hostname,
"port": port
})
self.client = Catalogue(service_url=f"{endpoint}{api_path}",
username=username,
password=password)
self.wms = WebMapService(url=f"{endpoint}{wms_path}",
version="1.3.0",
username=username,
password=password)
self.wfs = WebFeatureService(url=f"{endpoint}{wfs_path}",
version="2.0.0",
username=username,
password=password)
super().__init__(
server_id=server_id,
label=label,
hostname=hostname,
server_type=ServerType.GEOSERVER.value,
version=self._get_geoserver_version(),
)
def get_namespaces(self) -> List[str]:
"""
Gets all GeoServer workspace names as Namespace labels
:return: list of Namespace labels
"""
workspaces = []
for workspace in self.client.get_workspaces():
workspaces.append(workspace.name)
return workspaces
def get_namespace(self, namespace_reference: str) -> Dict[str, str]:
"""
Gets a specific workspace as a Namespace
Note: GeoServer workspaces do not support the concept of a title, a static substitute value is therefore used
Note: GeoServer workspaces do support the concept of a namespace, but it is not yet implemented [#28]
:param namespace_reference: Namespace (workspace) label (name)
:return: dictionary of Namespace information that can be made into a Namespace object
"""
workspace = self.client.get_workspace(name=namespace_reference)
if workspace is None:
raise KeyError(
f"Namespace [{namespace_reference}] not found in server [{self.label}]"
)
return {"label": workspace.name, "title": "-", "namespace": "-"}
def get_repositories(self) -> List[Tuple[str, str]]:
"""
Gets all GeoServer store names as Repository labels
:return: list of Repository:Namespace label tuples
"""
stores = []
# Passing workspaces here is a workaround for a bug in the get stores method where workspaces aren't specified.
# The method says all workspaces should be checked but the logic to do this is in the wrong place so none are.
for store in self.client.get_stores(
workspaces=self.client.get_workspaces()):
stores.append((store.name, store.workspace.name))
return stores
def get_repository(self, repository_reference: str,
namespace_reference: str) -> Dict[str, str]:
"""
Gets a specific store as a Repository
If a Namespace (workspace) label is specified the Repository must exist within that Namespace.
GeoServer store types are sometimes unsuitable or non-standard and so need to be mapped to a conventional value.
in the RepositoryType enum using the GeoServerRepositoryType enum.
Note: GeoServer stores do not support the concept of a title, a static substitute value is therefore used
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param repository_reference: Repository (store) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of repository information that can be made into a Repository object
"""
_store = self.client.get_store(name=repository_reference,
workspace=namespace_reference)
if _store is None:
raise KeyError(
f"Repository [{repository_reference}] not found in server [{self.label}]"
)
store = {
"label":
_store.name,
"title":
"-",
"repository_type":
RepositoryType[GeoServerRepositoryType(str(
_store.type).lower()).name].value,
"namespace_label":
_store.workspace.name,
}
if hasattr(_store, "description") and _store.description is not None:
store["title"] = _store.description
if (store["repository_type"] == RepositoryType.POSTGIS.value
or store["repository_type"] == RepositoryType.ORACLE.value):
store["hostname"] = _store.connection_parameters["host"]
store["database"] = _store.connection_parameters["database"]
store["schema"] = _store.connection_parameters["schema"]
return store
def get_styles(self) -> List[Tuple[str, Optional[str]]]:
"""
Gets all GeoServer style names as Style labels
Python's None value will be used to represent the Namespace of global styles (i.e that don't have a Namespace
(workspace)).
:return: list of Style:Namespace label tuples
"""
styles = []
for _style in self.client.get_styles():
styles.append((_style.name, _style.workspace))
return styles
def get_style(self,
style_reference: str,
namespace_reference: str = None) -> Dict[str, str]:
"""
Gets a specific style as a Style
If a Namespace (workspace) label is specified the Style must exist within that Namespace.
Note: GeoServer styles do support the concept of a title, but it is not exposed through the admin API so a
static substitute value is therefore used
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param style_reference: Style (style) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of style information that can be made into a Style object
"""
_style = self.client.get_style(name=style_reference,
workspace=namespace_reference)
_type = str(_style.style_format).lower()
if _type == "sld10":
_type = "sld"
style = {
"label": _style.name,
"title": "-",
"style_type": _type,
}
if hasattr(_style, "workspace") and _style.workspace is not None:
style["namespace_label"] = _style.workspace
return style
def get_layers(self) -> List[str]:
"""
Gets all GeoServer layer names as Layer labels
:return: list of Layer labels
"""
layers = []
for _layer in self.client.get_layers():
layers.append(_layer.name)
return layers
def get_layer(
self, layer_reference: str
) -> Dict[str, Union[Optional[str], List[str], List[Tuple[
str, Optional[str]]]]]:
"""
Gets a specific layer as a Layer
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param layer_reference: Layer (layer) label (name)
:return: dictionary of layer information that can be made into a Layer object
"""
_layer = self.client.get_layer(name=layer_reference)
layer = {
"label":
_layer.resource.name,
"title":
_layer.resource.title,
"layer_type":
str(_layer.type).lower(),
"geometry_type":
None,
"services": [],
"table_view":
None,
"namespace_label":
_layer.resource.workspace.name,
"repository_label":
_layer.resource.store.name,
"style_labels":
[(_layer.default_style.name, _layer.default_style.workspace)],
}
if layer_reference in list(
self.wms.contents
) or f"{_layer.resource.workspace.name}:{layer_reference}" in list(
self.wms.contents):
layer["services"].append(LayerService.WMS.value)
if layer_reference in list(
self.wfs.contents
) or f"{_layer.resource.workspace.name}:{layer_reference}" in list(
self.wfs.contents):
layer["services"].append(LayerService.WFS.value)
# WFS lookups don't seem to mind if the layer is namespaced or not
_properties = self.wfs.get_schema(layer_reference)
if "geometry" in _properties and isinstance(
_properties["geometry"], str):
try:
layer["geometry_type"] = LayerGeometry[
GeoServerLayerGeometry(str(
_properties["geometry"])).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['geometry']}] for layer {layer_reference} not mapped to "
f"LayerGeometry enum.")
elif "properties" in _properties:
for geometry_column_name in GeoServerGeometryColumnNames:
if geometry_column_name.value in _properties[
"properties"].keys():
try:
layer["geometry_type"] = LayerGeometry[
GeoPropertyGeoServerLayerGeom(
str(_properties["properties"][
geometry_column_name.value])
).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['properties'][geometry_column_name.value]}] for layer "
f"{layer_reference} in column '{geometry_column_name.value}' not mapped to "
f"LayerGeometry enum.")
if (str(_layer.resource.store.type).lower()
== RepositoryType.POSTGIS.value
or str(_layer.resource.store.type).lower()
== RepositoryType.ORACLE.value):
layer["table_view"] = _layer.resource.native_name
return layer
def get_layer_groups(self) -> List[Tuple[str, Optional[str]]]:
"""
Gets all GeoServer layer group names as LayerGroup labels
Python's None value will be used to represent the Namespace of global layer groups (i.e that don't have a
Namespace (workspace)).
:return: list of LayerGroup:Namespace label tuples
"""
layer_groups = []
for _layer_group in self.client.get_layergroups(
workspaces=self.client.get_workspaces()):
layer_groups.append((_layer_group.name, _layer_group.workspace))
return layer_groups
def get_layer_group(
self, layer_group_reference: str, namespace_reference: str
) -> Dict[str, Union[Optional[str], List[str], List[Tuple[
str, Optional[str]]]]]:
"""
Gets a specific layer group as a LayerGroup
If a Namespace (workspace) label is specified the LayerGroup must exist within that Namespace.
Note: Names (labels) will be returned for related components instead of identifiers or complete objects [#33]
:param layer_group_reference: LayerGroup (layer group) label (name)
:param namespace_reference: Namespace (store) label (name)
:return: dictionary of layer group information that can be made into a LayerGroup object
"""
_layer_group = self.client.get_layergroup(
name=layer_group_reference, workspace=namespace_reference)
layer_group = {
"label": _layer_group.name,
"title": _layer_group.title,
"services": [],
"namespace_label": _layer_group.workspace,
"layer_labels": [],
"style_labels": [],
}
for layer_label in _layer_group.layers:
layer_label = layer_label.split(":")
if len(layer_label) == 2:
layer_group["layer_labels"].append(
(layer_label[1], layer_label[0]))
elif len(layer_label) == 1:
layer_group["layer_labels"].append((layer_label[0], None))
if f"{namespace_reference}:{layer_group_reference}" in list(
self.wms.contents):
layer_group["services"].append(LayerService.WMS.value)
if f"{namespace_reference}:{layer_group_reference}" in list(
self.wfs.contents):
layer_group["services"].append(LayerService.WFS.value)
_properties = self.wfs.get_schema(
f"{namespace_reference}:{layer_group_reference}")
try:
layer_group["geometry_type"] = LayerGeometry[
GeoServerLayerGeometry(str(
_properties["geometry"])).name].value
except ValueError:
raise ValueError(
f"Geometry [{_properties['geometry']}] not mapped to LayerGeometry enum."
)
for style_label in _layer_group.styles:
if style_label is not None:
style_label = style_label.split(":")
if len(style_label) == 2 and (
style_label[1],
style_label[0]) not in layer_group["style_labels"]:
layer_group["style_labels"].append(
(style_label[1], style_label[0]))
if len(style_label) == 1 and (
style_label[0],
None) not in layer_group["style_labels"]:
layer_group["style_labels"].append((style_label[0], None))
return layer_group
def _get_geoserver_version(self) -> str:
"""
Gets the GeoServer version
:return: GeoServer version string
"""
return self.client.get_version()
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assert_(self.cat.get_layer('import') is None)
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assert_(self.cat.get_layer('import') is not None)
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
self.cat.delete(ds)
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(
ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
def testCoverageStoreCreate(self):
ds = self.cat.create_coveragestore2("coverage_gsconfig")
ds.data_url = "file:data/mytiff.tiff"
self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(
['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before,
str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after,
str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before,
str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after,
str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats,
str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after,
str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
# sanitize the layer name - validation will fail on newer geoservers
name = layer.replace(':', '_')
new_layer = self.cat.create_wmslayer(wmstest,
wmsstore,
name,
nativeName=layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
changed_layer = added_layers[0]
self.assertEqual(True, changed_layer.advertised)
self.assertEqual(True, changed_layer.enabled)
changed_layer.advertised = False
changed_layer.enabled = False
self.cat.save(changed_layer)
self.cat._cache.clear()
changed_layer = wmsstore.get_resources()[0]
changed_layer.fetch()
self.assertEqual(False, changed_layer.advertised)
self.assertEqual(False, changed_layer.enabled)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars,
sf)
self.assert_(
self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError, lambda: self.cat.create_featurestore(
"states_test", shapefile_plus_sidecars, sf))
self.assertRaises(
UploadError, lambda: self.cat.create_coveragestore(
"states_raster_test", shapefile_plus_sidecars, sf))
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf))
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
# TODO: Uploading WorldImage file no longer works???
# ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
# self.assertRaises(
# ConflictingDataError,
# lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# )
self.assertRaises(
UploadError, lambda: self.cat.create_featurestore(
"Pk50095_vector", tiffdata, sf))
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff))
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles
if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred",
open("test/ted.sld").read(),
overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assert_(f is not None)
self.assertEqual(f.name, fred.name)
def testWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("jed",
open("test/fred.sld").read(),
workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assert_(jed is None)
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
self.cat.create_style("jed",
open("test/ted.sld").read(),
overwrite=True,
workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assert_(self.cat.get_style("jed", workspace="topp") is None)
# attempt creating new style
self.cat.create_style("jed",
open("test/fred.sld").read(),
workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assert_(f is not None)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("ned",
open("test/fred.sld").read(),
overwrite=True,
workspace="topp")
self.cat.create_style("zed",
open("test/ted.sld").read(),
overwrite=True,
workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assert_(ned is not None)
self.assert_(zed is not None)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def testTimeDimension(self):
sf = self.cat.get_workspace("sf")
files = shapefile_and_friends(
os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date"))
self.cat.create_featurestore("boxes_with_end_date", files, sf)
get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer(
'boxes_with_end_date').resource
# configure time as LIST
resource = get_resource()
timeInfo = DimensionInfo("time",
"true",
"LIST",
None,
"ISO8601",
None,
attribute="date")
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual("date", timeInfo.attribute)
self.assertEqual("ISO8601", timeInfo.units)
# disable time dimension
timeInfo = resource.metadata['time']
timeInfo.enabled = False
# since this is an xml property, it won't get written unless we modify it
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(False, timeInfo.enabled)
# configure with interval, end_attribute and enable again
timeInfo.enabled = True
timeInfo.presentation = 'DISCRETE_INTERVAL'
timeInfo.resolution = '3 days'
timeInfo.end_attribute = 'enddate'
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(True, timeInfo.enabled)
self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation)
self.assertEqual('3 days', timeInfo.resolution_str())
self.assertEqual('enddate', timeInfo.end_attribute)
def testImageMosaic(self):
# testing the mosaic creation
name = 'cea_mosaic'
data = open('test/data/mosaic/cea.zip', 'rb')
self.cat.create_imagemosaic(name, data)
# get the layer resource back
self.cat._cache.clear()
resource = self.cat.get_layer(name).resource
self.assert_(resource is not None)
# delete granule from mosaic
coverage = name
store = name
granule_id = name + '.1'
self.cat.mosaic_delete_granule(coverage, store, granule_id)
class GeoserverHelper:
def __init__(self, geoserverUrl = "", geoserverUserName = "", geoserverPW = "", geoserverWorkSpace = "",
postgreIP = "", postgreUserName = "", postgrePW = ""):
"""use the constructor given arguments if used"""
self.geoserverUrl = geoserverUrl if geoserverUrl != "" else GEOSERVERURL
self.geoserverUserName = geoserverUserName if geoserverUserName != "" else GEOSERVERUSERNAME
self.geoserverPW = geoserverPW if geoserverPW != "" else GEOSERVERPW
self.geoserverWorkSpace = geoserverWorkSpace if geoserverWorkSpace != "" else "crc"
self.postgreIP = postgreIP if postgreIP != "" else POSTGREIP
self.postgreUserName = postgreUserName if postgreUserName != "" else POSTGREUSERNAME
self.postgrePW = postgrePW if postgrePW != "" else POSTGREPW
if self.geoserverUrl[-1] != '/':
raise Exception("GeoserverUrl must end with a slash ('/')")
self.catalog = Catalog(self.geoserverUrl+"rest/")
self.catalog.http.add_credentials(self.geoserverUserName,self.geoserverPW)
try:
workspaces = self.catalog.get_workspaces()
except:
e = sys.exc_info()[0]
print e
self.cWorkSpace = self.catalog.get_workspace(self.geoserverWorkSpace)
def getLayers(self):
return self.cWorkSpace.catalog.get_layers()
def insertShapeIntoPostGis(self, shapeFile, databaseName, tableName, encoding=3857):
'''returns the returnCode of the execution of the insert script, e.g:
helper.insertShapeIntoPostGis('/home/c815/gsTest/test.shp','crc','testingHelper2')'''
if not os.path.isfile(shapeFile):
print "Shape file not found"
return -1
cmds = "PGPASSWORD={pgPW} ./createWSFTFromSHP.sh -s {shapeFileF} -d {databaseNameF} -t {tableNameF} -u {postgreUsername} -i {postgreIP}".format(pgPW=self.postgrePW,
shapeFileF=shapeFile,databaseNameF=databaseName, tableNameF=tableName, postgreUsername=self.postgreUserName, postgreIP=self.postgreIP)
return subprocess.call(cmds, shell=True)
def uploadShapeFile(self, shapeFile, storeName):
shpPlusSidcars = geoserver.util.shapefile_and_friends(shapeFile[:-3])
shpPlusSidcars
self.ft = self.catalog.create_featurestore(storeName, shpPlusSidcars, self.cWorkSpace)
def getStyles(self):
return self.catalog.get_styles()
def uploadStyleFile(self, sldFile, styleName, overWrite, workSpace = None):
f = open(sldFile,'r')
styleSrc = f.read()
uploadStyle(styleSrc, styleName, overWrite, workSpace)
f.close()
def uploadStyle(self, sldSrc, styleName, overWrite, workSpace = None):
self.catalog.create_style(styleName,sldSrc, overWrite, workSpace)
def publishPostGISLayer(self,postGISLayerName, storeName, crs='EPSG:3857'):
'''cat.publish_featuretype('testingstuff',crcStore,native_crs='EPSG:3857')'''
store = self.catalog.get_store(storeName)
if store != None:
self.catalog.publish_featuretype(postGISLayerName, store, crs)
def setDefaultStyleForLayer(self, layerName, styleName):
l = self.catalog.get_layer(layerName)
sNames = [ i.name for i in self.getStyles() ]
if styleName not in sNames:
split = styleName.split(':')
if len(split) == 2:
workSpace = styleName.split(':')[0]
newStyleName = styleName.split(':')[1]
else:
return -1
style = self.catalog.get_style(newStyleName, workSpace)
if style is None:
return -1
if l != None:
l._set_default_style(styleName)
self.catalog.save(l)
return 0
def createPostGISDataStore(self, storeName, postGisPassword, postGisUser, postGisHost,
postGisDatabase, workSpace = None):
#check if connection parameter are valid
try:
conn = psycopg2.connect("dbname='{dbName}' user='******' host='{Host}' password='******'".format(dbName=postGisDatabase,
dbUser=postGisUser,
Host=postGisHost,
password=postGisPassword))
except:
return False
w = self.catalog.create_datastore(storeName, workSpace)
template = Template("""{'validate connections': 'true', 'port': '5432',
'Support on the fly geometry simplification': 'true',
'create database': 'false', 'dbtype': 'postgis',
'Connection timeout': '20', 'namespace': 'http://www.crcproject.com',
'Max connection idle time': '300', 'Expose primary keys': 'false',
'min connections': '1', 'Max open prepared statements':'50',
'passwd': '$passwd',
'encode functions': 'false',
'max connections': '10', 'Evictor tests per run': '3', 'Loose bbox': 'true',
'Evictor run periodicity': '300', 'Estimated extends': 'true',
'database': '$database',
'fetch size': '1000', 'Test while idle': 'true',
'host': '$host',
'preparedStatements': 'false', 'schema': 'public',
'user': '******'}""")
dic = ast.literal_eval(template.substitute(
passwd=postGisPassword,
user=postGisUser,
host=postGisHost,
database=postGisDatabase))
#'passwd': 'crypt1:Bsaz2AUI8T+6Pj43krA7kg==',
#'user': '******'}
#'database': 'crc',
#'host': 'localhost',
w.connection_parameters = dic
self.catalog.save(w)
return True
def gs_slurp(ignore_errors=True, verbosity=1, console=None, owner=None, workspace=None, store=None, filter=None):
"""Configure the layers available in GeoServer in GeoNode.
It returns a list of dictionaries with the name of the layer,
the result of the operation and the errors and traceback if it failed.
"""
if console is None:
console = open(os.devnull, 'w')
if verbosity > 1:
print >> console, "Inspecting the available layers in GeoServer ..."
url = "%srest" % settings.GEOSERVER_BASE_URL
cat = Catalog(url, _user, _password)
if workspace is not None:
workspace = cat.get_workspace(workspace)
resources = cat.get_resources(workspace=workspace)
elif store is not None:
store = cat.get_store(store)
resources = cat.get_resources(store=store)
else:
resources = cat.get_resources(workspace=workspace)
if filter:
resources = [k for k in resources if filter in k.name]
# filter out layers explicitly disabled by geoserver
resources = [k for k in resources if k.enabled == "true"]
# TODO: Should we do something with these?
# i.e. look for matching layers in GeoNode and also disable?
disabled_resources = [k for k in resources if k.enabled == "false"]
number = len(resources)
if verbosity > 1:
msg = "Found %d layers, starting processing" % number
print >> console, msg
output = []
for i, resource in enumerate(resources):
name = resource.name
store = resource.store
workspace = store.workspace
try:
# Avoid circular import problem
from geonode.layers.models import Layer
layer, created = Layer.objects.get_or_create(name=name, defaults = {
"workspace": workspace.name,
"store": store.name,
"storeType": store.resource_type,
"typename": "%s:%s" % (workspace.name, resource.name),
"title": resource.title or 'No title provided',
"abstract": resource.abstract or 'No abstract provided',
"owner": owner,
"uuid": str(uuid.uuid4())
})
layer.save()
except Exception, e:
if ignore_errors:
status = 'failed'
exception_type, error, traceback = sys.exc_info()
else:
if verbosity > 0:
msg = "Stopping process because --ignore-errors was not set and an error was found."
print >> sys.stderr, msg
raise Exception('Failed to process %s' % resource.name, e), None, sys.exc_info()[2]
else:
if created:
layer.set_default_permissions()
status = 'created'
else:
status = 'updated'
msg = "[%s] Layer %s (%d/%d)" % (status, name, i+1, number)
info = {'name': name, 'status': status}
if status == 'failed':
info['traceback'] = traceback
info['exception_type'] = exception_type
info['error'] = error
output.append(info)
if verbosity > 0:
print >> console, msg
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'],
username=GSPARAMS['GSUSER'],
password=GSPARAMS['GSPASSWORD'])
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assertIsNone(self.cat.get_layer('import'))
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assertIsNotNone(self.cat.get_layer('import'))
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
try:
if lyr:
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
if ds:
self.cat.delete(ds)
except Exception:
pass
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(
ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
@drop_table('import2')
def testVirtualTables(self):
ds = self.cat.create_datastore("gsconfig_import_test2")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test2")
self.cat.add_data_to_store(
ds, "import2", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
store = self.cat.get_store("gsconfig_import_test2")
geom = JDBCVirtualTableGeometry('the_geom', 'MultiPolygon', '4326')
ft_name = 'my_jdbc_vt_test'
epsg_code = 'EPSG:4326'
sql = "select * from import2 where 'STATE_NAME' = 'Illinois'"
keyColumn = None
parameters = None
jdbc_vt = JDBCVirtualTable(ft_name, sql, 'false', geom, keyColumn,
parameters)
self.cat.publish_featuretype(ft_name,
store,
epsg_code,
jdbc_virtual_table=jdbc_vt)
# DISABLED; this test works only in the very particular case
# "mytiff.tiff" is already present into the GEOSERVER_DATA_DIR
# def testCoverageStoreCreate(self):
# ds = self.cat.create_coveragestore2("coverage_gsconfig")
# ds.data_url = "file:test/data/mytiff.tiff"
# self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211",
"http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual([
("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")
], rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(
['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEqual(set(rs.request_srs_list), srs_before,
str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(set(rs.request_srs_list), srs_after,
str(rs.request_srs_list))
# set and save response_srs_list
self.assertEqual(set(rs.response_srs_list), srs_before,
str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(set(rs.response_srs_list), srs_after,
str(rs.response_srs_list))
# set and save supported_formats
self.assertEqual(set(rs.supported_formats), formats,
str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(set(rs.supported_formats), formats_after,
str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/iowa/rainfall.cgi?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=WMS&" # noqa: E501
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(workspace=wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
# sanitize the layer name - validation will fail on newer geoservers
name = layer.replace(':', '_')
self.cat.create_wmslayer(wmstest, wmsstore, name, nativeName=layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
changed_layer = added_layers[0]
self.assertEqual(True, changed_layer.advertised)
self.assertEqual(True, changed_layer.enabled)
changed_layer.advertised = False
changed_layer.enabled = False
self.cat.save(changed_layer)
self.cat._cache.clear()
changed_layer = wmsstore.get_resources()[0]
changed_layer.fetch()
self.assertEqual(False, changed_layer.advertised)
self.assertEqual(False, changed_layer.enabled)
# Testing projection and projection policy changes
changed_layer.projection = "EPSG:900913"
changed_layer.projection_policy = "REPROJECT_TO_DECLARED"
self.cat.save(changed_layer)
self.cat._cache.clear()
layer = self.cat.get_layer(changed_layer.name)
self.assertEqual(layer.resource.projection_policy,
changed_layer.projection_policy)
self.assertEqual(layer.resource.projection, changed_layer.projection)
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.items():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars,
sf)
self.assertIsNotNone(self.cat.get_resource("states_test",
workspace=sf))
self.assertRaises(
ConflictingDataError, lambda: self.cat.create_featurestore(
"states_test", shapefile_plus_sidecars, sf))
self.assertRaises(
UploadError, lambda: self.cat.create_coveragestore(
"states_raster_test", shapefile_plus_sidecars, sf))
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf))
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assertIsNone(self.cat.get_layer("states_test"))
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
self.cat.create_coveragestore("Pk50095", tiffdata, sf)
self.assertIsNotNone(self.cat.get_resource("Pk50095", workspace=sf))
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf))
self.assertRaises(
UploadError, lambda: self.cat.create_featurestore(
"Pk50095_vector", tiffdata, sf))
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff))
self.cat.create_coveragestore_external_geotiff(
"Pk50095_ext", 'file:test/data/Pk50095.tif', sf)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = {
'title': 'Not the original attribution',
'width': '123',
'height': '321',
'href': 'http://www.georchestra.org',
'url':
'https://www.cigalsace.org/portail/cigal/documents/page/mentions-legales/Logo_geOrchestra.jpg',
'type': 'image/jpeg'
}
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = next(
(s for s in lyr.styles if s.name == "pophatch"))
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# check count before tests (upload)
count = len(self.cat.get_styles())
# upload new style, verify existence
with open("test/fred.sld") as fred_sld:
self.cat.create_style("fred", fred_sld.read())
fred = self.cat.get_style("fred")
self.assertIsNotNone(fred)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
with open("test/ted.sld") as ted_sld:
self.cat.create_style("fred", ted_sld.read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assertIsNotNone(fred)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assertIsNone(self.cat.get_style("fred"))
# attempt creating new style
with open("test/fred.sld") as fred_sld:
self.cat.create_style("fred", fred_sld.read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assertIsNotNone(f)
self.assertEqual(f.name, fred.name)
# compare count after upload
self.assertEqual(count + 1, len(self.cat.get_styles()))
# attempt creating a new style without "title"
with open("test/notitle.sld") as notitle_sld:
self.cat.create_style("notitle", notitle_sld.read())
notitle = self.cat.get_style("notitle")
self.assertEqual(None, notitle.sld_title)
def testWorkspaceStyles(self):
# upload new style, verify existence
with open("test/fred.sld") as fred_sld:
self.cat.create_style("jed", fred_sld.read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assertIsNone(jed)
jed = self.cat.get_style("jed", workspace="topp")
self.assertIsNotNone(jed)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assertIsNotNone(jed)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
with open("test/ted.sld") as ted_sld:
self.cat.create_style("jed",
ted_sld.read(),
overwrite=True,
workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertIsNotNone(jed)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assertIsNone(self.cat.get_style("jed", workspace="topp"))
# attempt creating new style
with open("test/fred.sld") as fred_sld:
self.cat.create_style("jed", fred_sld.read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assertIsNotNone(f)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
with open("test/fred.sld") as fred_sld:
self.cat.create_style("ned",
fred_sld.read(),
overwrite=True,
workspace="topp")
with open("test/fred.sld") as ted_sld:
self.cat.create_style("zed",
ted_sld.read(),
overwrite=True,
workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assertIsNotNone(ned)
self.assertIsNotNone(zed)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
if lyr.default_style is not None:
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assertIsNone(ws)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assertTrue(states.enabled)
states.enabled = False
self.assertFalse(states.enabled)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assertFalse(states.enabled)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assertTrue(states.enabled)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def testImageMosaic(self):
"""
Test case for Issue #110
"""
# testing the mosaic creation
name = 'cea_mosaic'
with open('test/data/mosaic/cea.zip', 'rb') as data:
self.cat.create_imagemosaic(name, data)
# get the layer resource back
self.cat._cache.clear()
resource = self.cat.get_layer(name).resource
self.assertIsNotNone(resource)
# delete granule from mosaic
coverage = name
store = self.cat.get_store(name)
granules = self.cat.list_granules(coverage, store)
self.assertEqual(1, len(granules['features']))
granule_id = name + '.1'
self.cat.mosaic_delete_granule(coverage, store, granule_id)
granules = self.cat.list_granules(coverage, store)
self.assertEqual(0, len(granules['features']))
"""
testing external Image mosaic creation
"""
name = 'cea_mosaic_external'
path = 'test/data/mosaic/external'
self.cat.create_imagemosaic(name, path, workspace='topp')
self.cat._cache.clear()
resource = self.cat.get_layer("external").resource
self.assertIsNotNone(resource)
# add granule to mosaic
granule_path = 'test/data/mosaic/granules/cea_20150102.tif'
self.cat.add_granule(granule_path, name, workspace='topp')
granules = self.cat.list_granules("external", name, 'topp')
self.assertEqual(2, len(granules['features']))
# add external granule to mosaic
granule_path = os.path.join(
os.getcwd(), 'test/data/mosaic/granules/cea_20150103.zip')
self.cat.add_granule(granule_path, name, workspace='topp')
granules = self.cat.list_granules("external", name, 'topp')
self.assertEqual(3, len(granules['features']))
# Delete store
store = self.cat.get_store(name)
self.cat.delete(store, purge=True, recurse=True)
self.cat._cache.clear()
def testTimeDimension(self):
sf = self.cat.get_workspace("sf")
files = shapefile_and_friends(
os.path.join(gisdata.GOOD_DATA, "time", "boxes_with_end_date"))
self.cat.create_featurestore("boxes_with_end_date", files, sf)
get_resource = lambda: self.cat._cache.clear() or self.cat.get_layer(
'boxes_with_end_date').resource # noqa: E501
# configure time as LIST
resource = get_resource()
timeInfo = DimensionInfo("time",
"true",
"LIST",
None,
"ISO8601",
None,
attribute="date")
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual("LIST", timeInfo.presentation)
self.assertEqual(True, timeInfo.enabled)
self.assertEqual("date", timeInfo.attribute)
self.assertEqual("ISO8601", timeInfo.units)
# disable time dimension
timeInfo = resource.metadata['time']
timeInfo.enabled = False
# since this is an xml property, it won't get written unless we modify it
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(False, timeInfo.enabled)
# configure with interval, end_attribute and enable again
timeInfo.enabled = True
timeInfo.presentation = 'DISCRETE_INTERVAL'
timeInfo.resolution = '3 days'
timeInfo.end_attribute = 'enddate'
resource.metadata = {'time': timeInfo}
self.cat.save(resource)
# and verify
resource = get_resource()
timeInfo = resource.metadata['time']
self.assertEqual(True, timeInfo.enabled)
self.assertEqual('DISCRETE_INTERVAL', timeInfo.presentation)
self.assertEqual('3 days', timeInfo.resolution_str())
self.assertEqual('enddate', timeInfo.end_attribute)
class GeoserverLayers(object):
def __init__(self, gs_apirest, username, password):
try:
self.__catalog = Catalog(gs_apirest, username=username, password=password)
except Exception as err:
print("Geoserver API Rest Error: {0}".format(err))
def __createSldStyle(self, sldpath, stylename, ovwrt=False):
try:
styles_list = [st.name for st in self.__catalog.get_styles()]
if ovwrt or not stylename in styles_list:
with open(sldpath) as f:
self.__catalog.create_style(stylename, f.read(), overwrite=ovwrt)
else:
print("This style already exists...")
except Exception as err:
print("Geoserver API Rest Error creating new style: {0}".format(err))
def __rmvResources(self, rsrc):
"""
Removing resources
"""
self.__catalog.delete(rsrc)
self.__catalog.reload()
def rmvStyle(self, stylename):
"""
Remove style
"""
try:
style = self.__catalog.get_style(stylename)
self.__rmvResources(style)
except Exception as err:
print("Geoserver API Rest Error removing style: {0}".format(err))
def rmvDataStore(self, ds_store):
"""
Remove DataStore (with his layer)
"""
try:
lay_rm = self.__catalog.get_layer(ds_store)
self.__rmvResources(lay_rm)
str_rm = self.__catalog.get_store(ds_store)
self.__rmvResources(str_rm)
except Exception as err:
print("Geoserver API Rest Error removing data store: {0}".format(err))
def createGeoserverWMSLayer(self, data, ws_name, ds_name, stylename, sldpath, debug=False):
"""
Create Geoserver WMS layer
Status codes:
2 : "Geoserver layer successfully created"
-1 : "Workspace does not exist"
-2 : "Datastore already exists"
-3 : "Error creating Geoserver layer"
-4 : "File missing"
"""
try:
if not self.__catalog.get_workspace(ws_name):
print("Workspace does not exist")
return -1
ds_list = [i.name for i in self.__catalog.get_stores()]
if ds_name in ds_list:
print("Datastore already exists")
return -2
ds = self.__catalog.create_datastore(ds_name, ws_name)
ft = self.__catalog.create_featurestore(ds_name, workspace=ws_name,data=data)
print("Geoserver layer successfully created...")
self.__createSldStyle(sldpath, stylename)
lyr = self.__catalog.get_layer(ds_name)
lyr.enabled = True
lyr.default_style = stylename
self.__catalog.save(lyr)
if debug:
rsrc = self.__catalog.get_resource(ds_name, workspace=ws_name)
print("Data Store XML: {}".format(rsrc.href))
return 2
except Exception as err:
print("Geoserver API Rest Error creating new layer: {0}".format(err))
return -3
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog(GSPARAMS['GSURL'], username=GSPARAMS['GSUSER'], password=GSPARAMS['GSPASSWORD'])
def testAbout(self):
about_html = self.cat.about()
self.assertTrue('<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile", self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states", self.cat.get_resource("states", states).name)
self.assertEqual("states", self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title,
states.abstract,
states.native_bbox,
states.latlon_bbox,
states.projection,
states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testResourcesUpdate(self):
res_dest = self.cat.get_resources()
count = 0
for rd in res_dest:
# only wms layers
if rd.resource_type != "wmsLayer": continue
# looking for same name
ro = self.cat.get_resource(rd.name)
if ro is not None:
rd.title = ro.title
rd.abstract = ro.abstract
rd.keywords = ro.keywords
rd.projection = ro.projection
rd.native_bbox = ro.native_bbox
rd.latlon_bbox = ro.latlon_bbox
rd.projection_policy = ro.projection_policy
rd.enabled = ro.enabled
rd.advertised = ro.advertised
rd.metadata_links = ro.metadata_links or None
self.cat.save(rd)
self.cat.reload()
# print "Updated layer: " + rd.name
count += 1
# print "Total updated layers: " + str(count)
def testLayers(self):
expected = set(["Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies", "tasmania_state_boundaries",
"tasmania_cities", "states", "poly_landmarks", "tiger_roads", "poi",
"giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles), set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
# Try to create a new Layer Group into the "topp" workspace
self.assert_(self.cat.get_workspace("topp") is not None)
tas2 = self.cat.create_layergroup("tasmania_reloaded", tas.layers, workspace = "topp")
self.cat.save(tas2)
self.assert_(self.cat.get_layergroup("tasmania_reloaded") is None)
self.assert_(self.cat.get_layergroup("tasmania_reloaded", "topp") is not None)
tas2 = self.cat.get_layergroup("tasmania_reloaded", "topp")
self.assert_("tasmania_reloaded", tas2.name)
self.assert_(isinstance(tas2, LayerGroup))
self.assertEqual(tas2.workspace, "topp", tas2.workspace)
self.assertEqual(tas2.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas2.layers)
self.assertEqual(tas2.styles, [None, None, None, None], tas2.styles)
def testStyles(self):
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld", self.cat.get_style("population").filename)
self.assertEqual("population", self.cat.get_style("population").sld_name)
self.assert_(self.cat.get_style('non-existing-style') is None)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
try:
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.fail('expected exception')
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever', fre.message)
try:
self.cat.get_resource(workspace="best workspace ever",
store="best store ever",
name="best resource ever")
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever', fre.message)
class ModifyingTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testFeatureTypeSave(self):
# test saving round trip
rs = self.cat.get_resource("bugsites")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
enabled = rs.enabled
# Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(new_abstract, rs.abstract)
self.assertEqual(enabled, rs.enabled)
# Change keywords on server
rs.keywords = ["bugsites", "gsconfig"]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(["bugsites", "gsconfig"], rs.keywords)
self.assertEqual(enabled, rs.enabled)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("bugsites")
self.assertEqual(old_abstract, rs.abstract)
def testDataStoreCreate(self):
ds = self.cat.create_datastore("vector_gsconfig")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
def testPublishFeatureType(self):
# Use the other test and store creation to load vector data into a database
# @todo maybe load directly to database?
try:
self.testDataStoreCreateAndThenAlsoImportData()
except FailedRequestError:
pass
try:
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
ds = self.cat.get_store("gsconfig_import_test")
# make sure it's gone
self.assert_(self.cat.get_layer('import') is None)
self.cat.publish_featuretype("import", ds, native_crs="EPSG:4326")
# and now it's not
self.assert_(self.cat.get_layer('import') is not None)
finally:
# tear stuff down to allow the other test to pass if we run first
ds = self.cat.get_store("gsconfig_import_test")
lyr = self.cat.get_layer('import')
# Delete the existing layer and resource to allow republishing.
self.cat.delete(lyr)
self.cat.delete(lyr.resource)
self.cat.delete(ds)
def testDataStoreModify(self):
ds = self.cat.get_store("sf")
self.assertFalse("foo" in ds.connection_parameters)
ds.connection_parameters = ds.connection_parameters
ds.connection_parameters["foo"] = "bar"
orig_ws = ds.workspace.name
self.cat.save(ds)
ds = self.cat.get_store("sf")
self.assertTrue("foo" in ds.connection_parameters)
self.assertEqual("bar", ds.connection_parameters["foo"])
self.assertEqual(orig_ws, ds.workspace.name)
@drop_table('import')
def testDataStoreCreateAndThenAlsoImportData(self):
ds = self.cat.create_datastore("gsconfig_import_test")
ds.connection_parameters.update(**DBPARAMS)
self.cat.save(ds)
ds = self.cat.get_store("gsconfig_import_test")
self.cat.add_data_to_store(ds, "import", {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
})
def testCoverageStoreCreate(self):
ds = self.cat.create_coveragestore2("coverage_gsconfig")
ds.data_url = "file:data/mytiff.tiff"
self.cat.save(ds)
def testCoverageStoreModify(self):
cs = self.cat.get_store("sfdem")
self.assertEqual("GeoTIFF", cs.type)
cs.type = "WorldImage"
self.cat.save(cs)
cs = self.cat.get_store("sfdem")
self.assertEqual("WorldImage", cs.type)
# not sure about order of test runs here, but it might cause problems
# for other tests if this layer is misconfigured
cs.type = "GeoTIFF"
self.cat.save(cs)
def testCoverageSave(self):
# test saving round trip
rs = self.cat.get_resource("Arc_Sample")
old_abstract = rs.abstract
new_abstract = "Not the original abstract"
# # Change abstract on server
rs.abstract = new_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(new_abstract, rs.abstract)
# Restore abstract
rs.abstract = old_abstract
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(old_abstract, rs.abstract)
# Change metadata links on server
rs.metadata_links = [("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")]
enabled = rs.enabled
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEqual(
[("text/xml", "TC211", "http://example.com/gsconfig.test.metadata")],
rs.metadata_links)
self.assertEqual(enabled, rs.enabled)
srs_before = set(['EPSG:4326'])
srs_after = set(['EPSG:4326', 'EPSG:3785'])
formats = set(['ARCGRID', 'ARCGRID-GZIP', 'GEOTIFF', 'PNG', 'GIF', 'TIFF'])
formats_after = set(["PNG", "GIF", "TIFF"])
# set and save request_srs_list
self.assertEquals(set(rs.request_srs_list), srs_before, str(rs.request_srs_list))
rs.request_srs_list = rs.request_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.request_srs_list), srs_after, str(rs.request_srs_list))
# set and save response_srs_list
self.assertEquals(set(rs.response_srs_list), srs_before, str(rs.response_srs_list))
rs.response_srs_list = rs.response_srs_list + ['EPSG:3785']
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.response_srs_list), srs_after, str(rs.response_srs_list))
# set and save supported_formats
self.assertEquals(set(rs.supported_formats), formats, str(rs.supported_formats))
rs.supported_formats = ["PNG", "GIF", "TIFF"]
self.cat.save(rs)
rs = self.cat.get_resource("Arc_Sample")
self.assertEquals(set(rs.supported_formats), formats_after, str(rs.supported_formats))
def testWmsStoreCreate(self):
ws = self.cat.create_wmsstore("wmsstore_gsconfig")
ws.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
ws.type = "WMS"
self.cat.save(ws)
def testWmsLayer(self):
self.cat.create_workspace("wmstest", "http://example.com/wmstest")
wmstest = self.cat.get_workspace("wmstest")
wmsstore = self.cat.create_wmsstore("wmsstore", wmstest)
wmsstore.capabilitiesURL = "http://suite.opengeo.org/geoserver/ows?service=wms&version=1.1.1&request=GetCapabilities"
wmsstore.type = "WMS"
self.cat.save(wmsstore)
wmsstore = self.cat.get_store("wmsstore")
self.assertEqual(1, len(self.cat.get_stores(wmstest)))
available_layers = wmsstore.get_resources(available=True)
for layer in available_layers:
new_layer = self.cat.create_wmslayer(wmstest, wmsstore, layer)
added_layers = wmsstore.get_resources()
self.assertEqual(len(available_layers), len(added_layers))
def testFeatureTypeCreate(self):
shapefile_plus_sidecars = shapefile_and_friends("test/data/states")
expected = {
'shp': 'test/data/states.shp',
'shx': 'test/data/states.shx',
'dbf': 'test/data/states.dbf',
'prj': 'test/data/states.prj'
}
self.assertEqual(len(expected), len(shapefile_plus_sidecars))
for k, v in expected.iteritems():
self.assertEqual(v, shapefile_plus_sidecars[k])
sf = self.cat.get_workspace("sf")
self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
self.assert_(self.cat.get_resource("states_test", workspace=sf) is not None)
self.assertRaises(
ConflictingDataError,
lambda: self.cat.create_featurestore("states_test", shapefile_plus_sidecars, sf)
)
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster_test", shapefile_plus_sidecars, sf)
)
bogus_shp = {
'shp': 'test/data/Pk50095.tif',
'shx': 'test/data/Pk50095.tif',
'dbf': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("bogus_shp", bogus_shp, sf)
)
lyr = self.cat.get_layer("states_test")
self.cat.delete(lyr)
self.assert_(self.cat.get_layer("states_test") is None)
def testCoverageCreate(self):
tiffdata = {
'tiff': 'test/data/Pk50095.tif',
'tfw': 'test/data/Pk50095.tfw',
'prj': 'test/data/Pk50095.prj'
}
sf = self.cat.get_workspace("sf")
# TODO: Uploading WorldImage file no longer works???
# ft = self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# self.assert_(self.cat.get_resource("Pk50095", workspace=sf) is not None)
# self.assertRaises(
# ConflictingDataError,
# lambda: self.cat.create_coveragestore("Pk50095", tiffdata, sf)
# )
self.assertRaises(
UploadError,
lambda: self.cat.create_featurestore("Pk50095_vector", tiffdata, sf)
)
bogus_tiff = {
'tiff': 'test/data/states.shp',
'tfw': 'test/data/states.shx',
'prj': 'test/data/states.prj'
}
self.assertRaises(
UploadError,
lambda: self.cat.create_coveragestore("states_raster", bogus_tiff)
)
def testLayerSave(self):
# test saving round trip
lyr = self.cat.get_layer("states")
old_attribution = lyr.attribution
new_attribution = "Not the original attribution"
# change attribution on server
lyr.attribution = new_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(new_attribution, lyr.attribution)
# Restore attribution
lyr.attribution = old_attribution
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(old_attribution, lyr.attribution)
self.assertEqual(lyr.default_style.name, "population")
old_default_style = lyr.default_style
lyr.default_style = (s for s in lyr.styles if s.name == "pophatch").next()
lyr.styles = [old_default_style]
self.cat.save(lyr)
lyr = self.cat.get_layer("states")
self.assertEqual(lyr.default_style.name, "pophatch")
self.assertEqual([s.name for s in lyr.styles], ["population"])
def testStyles(self):
# upload new style, verify existence
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Fred", fred.sld_title)
# replace style, verify changes
self.cat.create_style("fred", open("test/ted.sld").read(), overwrite=True)
fred = self.cat.get_style("fred")
self.assert_(fred is not None)
self.assertEqual("Ted", fred.sld_title)
# delete style, verify non-existence
self.cat.delete(fred, purge=True)
self.assert_(self.cat.get_style("fred") is None)
# attempt creating new style
self.cat.create_style("fred", open("test/fred.sld").read())
fred = self.cat.get_style("fred")
self.assertEqual("Fred", fred.sld_title)
# verify it can be found via URL and check the name
f = self.cat.get_style_by_url(fred.href)
self.assert_(f is not None)
self.assertEqual(f.name, fred.name)
def testWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="blarny")
self.assert_(jed is None)
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
jed = self.cat.get_style("topp:jed")
self.assert_(jed is not None)
self.assertEqual("Fred", jed.sld_title)
# replace style, verify changes
self.cat.create_style("jed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assert_(jed is not None)
self.assertEqual("Ted", jed.sld_title)
# delete style, verify non-existence
self.cat.delete(jed, purge=True)
self.assert_(self.cat.get_style("jed", workspace="topp") is None)
# attempt creating new style
self.cat.create_style("jed", open("test/fred.sld").read(), workspace="topp")
jed = self.cat.get_style("jed", workspace="topp")
self.assertEqual("Fred", jed.sld_title)
# verify it can be found via URL and check the full name
f = self.cat.get_style_by_url(jed.href)
self.assert_(f is not None)
self.assertEqual(f.fqn, jed.fqn)
def testLayerWorkspaceStyles(self):
# upload new style, verify existence
self.cat.create_style("ned", open("test/fred.sld").read(), overwrite=True, workspace="topp")
self.cat.create_style("zed", open("test/ted.sld").read(), overwrite=True, workspace="topp")
ned = self.cat.get_style("ned", workspace="topp")
zed = self.cat.get_style("zed", workspace="topp")
self.assert_(ned is not None)
self.assert_(zed is not None)
lyr = self.cat.get_layer("states")
lyr.default_style = ned
lyr.styles = [zed]
self.cat.save(lyr)
self.assertEqual("topp:ned", lyr.default_style)
self.assertEqual([zed], lyr.styles)
lyr.refresh()
self.assertEqual("topp:ned", lyr.default_style.fqn)
self.assertEqual([zed.fqn], [s.fqn for s in lyr.styles])
def testWorkspaceCreate(self):
ws = self.cat.get_workspace("acme")
self.assertEqual(None, ws)
self.cat.create_workspace("acme", "http://example.com/acme")
ws = self.cat.get_workspace("acme")
self.assertEqual("acme", ws.name)
def testWorkspaceDelete(self):
self.cat.create_workspace("foo", "http://example.com/foo")
ws = self.cat.get_workspace("foo")
self.cat.delete(ws)
ws = self.cat.get_workspace("foo")
self.assert_(ws is None)
def testWorkspaceDefault(self):
# save orig
orig = self.cat.get_default_workspace()
neu = self.cat.create_workspace("neu", "http://example.com/neu")
try:
# make sure setting it works
self.cat.set_default_workspace("neu")
ws = self.cat.get_default_workspace()
self.assertEqual('neu', ws.name)
finally:
# cleanup and reset to the way things were
self.cat.delete(neu)
self.cat.set_default_workspace(orig.name)
ws = self.cat.get_default_workspace()
self.assertEqual(orig.name, ws.name)
def testFeatureTypeDelete(self):
pass
def testCoverageDelete(self):
pass
def testDataStoreDelete(self):
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
states.enabled = False
self.assert_(states.enabled == False)
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == False)
states.enabled = True
self.cat.save(states)
states = self.cat.get_store('states_shapefile')
self.assert_(states.enabled == True)
def testLayerGroupSave(self):
tas = self.cat.get_layergroup("tasmania")
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads', 'tasmania_cities'], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
tas.layers = tas.layers[:-1]
tas.styles = tas.styles[:-1]
self.cat.save(tas)
# this verifies the local state
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
# force a refresh to check the remote state
tas.refresh()
self.assertEqual(tas.layers, ['tasmania_state_boundaries', 'tasmania_water_bodies', 'tasmania_roads'], tas.layers)
self.assertEqual(tas.styles, [None, None, None], tas.styles)
def add_to_geoserver(self):
# Inspired (copied :) ) from https://groups.google.com/forum/#!msg/geonode-users/R-u57r8aECw/AuEpydZayfIJ # TODO : check license
# We connect to the catalog
gsUrl = settings.OGC_SERVER['default']['LOCATION'] + "rest"
gsUser = settings.OGC_SERVER['default']['USER']
gsPassword = settings.OGC_SERVER['default']['PASSWORD']
cat = Catalog(gsUrl, gsUser, gsPassword)
if cat is None:
raise Exception('unable to instantiate geoserver catalog')
# We get the workspace
ws = cat.get_workspace(settings.DEFAULT_WORKSPACE)
if ws is None:
raise Exception('workspace %s not found in geoserver' %
settings.DEFAULT_WORKSPACE)
# We get or create the datastore
store = cat.get_store(self.datastore_name, ws)
if store is None:
store = cat.create_datastore(self.datastore_name, ws)
store.connection_parameters.update(host="postgres",
port="5432",
database="postgres",
user="******",
passwd="postgres",
schema='offline_osm',
dbtype="postgis")
cat.save(store)
if store is None:
raise Exception('datastore %s not found in geoserver' %
self.datastore_name)
# We get or create each layer then register it into geonode
for layer in Command.layers:
layername = layer["name"]
self.stdout.write('Adding {} to geoserver...'.format(layername))
layer_exists = (not cat.get_layer(layername) is None)
if not layer_exists or not self.options['no_overwrite']:
self.stdout.write(
' layer does not exists or no_overwrite unset, we add...')
ft = cat.publish_featuretype(layername,
store,
'EPSG:4326',
srs='EPSG:4326')
if ft is None:
raise Exception('unable to publish layer %s' % layername)
ft.title = 'OpenStreetMap Offline - ' + layername.split(
'_')[-1]
ft.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.'
cat.save(ft)
self.stdout.write(
' adding the style for {}...'.format(layername))
# We get or create the workspace
style_path = os.path.join(os.path.dirname(__file__), '..',
'..', 'styles', layer["style_name"])
if not os.path.exists(style_path):
self.stdout.write(
' The file {} does not exist. No style will be applied for {}.'
.format(style_path, layername))
else:
cat.create_style(layer["style_name"],
open(style_path, 'r').read(),
overwrite=True,
workspace=settings.DEFAULT_WORKSPACE,
raw=True)
style = cat.get_style(layer["style_name"], ws)
if style is None:
raise Exception('style not found (%s)' %
(layer["style_name"]))
publishing = cat.get_layer(layername)
if publishing is None:
raise Exception('layer not found (%s)' % layerName)
publishing.default_style = style
cat.save(publishing)
self.stdout.write(
' registering {} into geonode...'.format(layername))
resource = cat.get_resource(layername, store, ws)
if resource is None:
raise Exception('resource not found (%s)' % layername)
layer, created = Layer.objects.get_or_create(name=layername)
layer.workspace = ws.name
layer.store = store.name
layer.storeType = store.resource_type
layer.typename = "%s:%s" % (ws.name.encode('utf-8'),
resource.name.encode('utf-8'))
layer.title = resource.title
layer.abstract = resource.abstract
layer.temporal_extent_start = self.import_timestamp
layer.temporal_extent_end = self.import_timestamp
try:
layer.save()
except TypeError as e:
# We ignore a specific error in Geonode resourcebase_post_save when settings.SITEURL has no host (relative URL)
self.stdout.write(
'TODO : report Geonode exception in resourcebase_post_save when settings.SITEURL has no host (relative URL) : '
+ str(e))
pass
if created:
layer.set_default_permissions()
else:
self.stdout.write(' layer already exists, we skip.')
# We get or create the laygroup
self.stdout.write('Adding layergroup to geoserver...')
layername = 'offline_osm'
layergroup_exists = (not cat.get_layergroup(layername) is None)
if not layer_exists or not self.options['no_overwrite']:
self.stdout.write(
' layer does not exists or no_overwrite unset, we add...')
layergroup = cat.get_layergroup(
layername, workspace=settings.DEFAULT_WORKSPACE)
if layergroup is None:
layergroup = cat.create_layergroup(
layername,
layers=layernames,
workspace=settings.DEFAULT_WORKSPACE)
if layergroup is None:
raise Exception('unable to publish layer %s' % layername)
layergroup.title = 'OpenStreetMap Offline'
layergroup.abstract = 'This is an automated extract of the OpenStreetMap database. It is available offline. It is intended to be used as a background layer, but the data can also server analysis purposes.'
cat.save(layergroup)
else:
self.stdout.write(' layergroup already exists, we skip.')
# TODO : can we add layergroups to Geonode ?
self.stdout.write(
"laygroup won't be added to geonode (not supported yet)")
def _register_cascaded_layers(service, owner=None):
"""
Register layers for a cascading WMS
"""
if service.type == 'WMS' or service.type == "OWS":
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user , _password)
# Can we always assume that it is geonode?
# Should cascading layers have a separate workspace?
cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE)
if cascade_ws is None:
cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE, 'cascade')
try:
store = cat.get_store(service.name,cascade_ws)
except Exception:
store = cat.create_wmsstore(service.name, cascade_ws)
wms = WebMapService(service.base_url)
layers = list(wms.contents)
count = 0
for layer in layers:
lyr = cat.get_resource(layer, store, cascade_ws)
if lyr is None:
if service.type in ["WMS","OWS"]:
resource = cat.create_wmslayer(cascade_ws, store, layer)
elif service.type == "WFS":
resource = cat.create_wfslayer(cascade_ws, store, layer)
if resource:
cascaded_layer, created = Layer.objects.get_or_create(
typename = "%s:%s" % (cascade_ws.name, resource.name),
defaults = {
"name": resource.name,
"workspace": cascade_ws.name,
"store": store.name,
"storeType": store.resource_type,
"title": resource.title or 'No title provided',
"abstract": resource.abstract or 'No abstract provided',
"owner": None,
"uuid": str(uuid.uuid4())
})
if created:
cascaded_layer.save()
if cascaded_layer is not None and cascaded_layer.bbox is None:
cascaded_layer._populate_from_gs(gs_resource=resource)
cascaded_layer.set_default_permissions()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service,
typename=cascaded_layer.name
)
service_layer.layer = cascaded_layer
service_layer.title=cascaded_layer.title,
service_layer.description=cascaded_layer.abstract,
service_layer.styles=cascaded_layer.styles
service_layer.save()
count += 1
else:
logger.error("Resource %s from store %s could not be saved as layer" % (layer, store.name))
message = "%d Layers Registered" % count
return_dict = {'status': 'ok', 'msg': message }
return HttpResponse(json.dumps(return_dict),
mimetype='application/json',
status=200)
elif service.type == 'WCS':
return HttpResponse('Not Implemented (Yet)', status=501)
else:
return HttpResponse('Invalid Service Type', status=400)
class Client:
def __init__(self, geoserver, username, password):
self.restserver = urlparse.urljoin(geoserver, 'rest/')
self.wmsserver = urlparse.urljoin(geoserver, 'wms')
self.username = username
self.password = password
self.catalog = Catalog(self.restserver, self.username, self.password)
self.tempDir = tempfile.mkdtemp()
self.resource = None
self.layer = None
self.layerName = None
logging.basicConfig(
format="%(asctime)-15s %(name)-10s %(levelname)-7s : %(message)s",
level=logging.WARN)
self.logger = logging.getLogger("gsclient")
self.logger.setLevel(logging.DEBUG)
# setup logging for the gsclient
logging.getLogger('pyclowder').setLevel(logging.DEBUG)
logging.getLogger('__main__').setLevel(logging.DEBUG)
## this method assume that there is 1 store per layer
def getResourceByStoreName(self, storename, workspace):
if self.resource != None:
self.logger.debug("resource instance found; no need to fetch")
return self.resource
self.logger.debug("catalog.get_store called")
store = self.catalog.get_store(storename, workspace)
self.logger.debug("catalog.get_resources called based on store")
resources = self.catalog.get_resources(store=store)
self.logger.debug("fetched resources from server")
if resources == None:
return None
else:
self.resource = resources[0]
return self.resource
def getLayers(self):
layers = self.catalog.get_layers()
return layers
def getLayerByStoreName(self, storename):
self.logger.debug("getLayerbystore name started")
layers = self.catalog.get_layers()
for layer in layers:
if layer.resource.store.name == storename:
self.logger.debug("found the layer by store name")
return layer
return None
def getLayerByResource(self, resource):
if self.layer != None:
self.logger.debug("layer instance found; no need to fetch")
return self.layer
self.logger.debug("get Layer by Resource started...")
layers = self.catalog.get_layers(resource)
self.logger.debug("fetched layers from the server")
if layers == None:
return None
else:
self.layer = layers[0]
return self.layer
def mintMetadataWithoutGeoserver(self, workspace, filename, extent):
self.logger.debug("Creating wms metadata ... ")
metadata = {}
wmsLayerName = workspace + ':' + filename
metadata['WMS Layer Name'] = wmsLayerName
metadata['WMS Service URL'] = self.wmsserver
metadata[
'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng'
self.logger.debug('[DONE]')
return metadata
def mintMetadata(self, workspace, storename, extent):
self.logger.debug("Creating wms metadata ... ")
metadata = {}
layername = None
if self.layerName == None:
if self.layer == None:
self.logger.debug("getResourceByStoreName..")
resource = self.getResourceByStoreName(storename, workspace)
self.logger.debug("getLayerByResource ...")
layer = self.getLayerByResource(resource)
#layername = layer.name
self.logger.debug("done getting layer name")
if layer == None:
self.logger.debug('No layer found [DONE]')
return metadata
else:
layername = layer.name
else:
layername = self.layer.name
self.layerName = self.layer.name
else:
layername = self.layerName
# generate metadata
wmsLayerName = workspace + ':' + layername
metadata['WMS Layer Name'] = wmsLayerName
metadata['WMS Service URL'] = self.wmsserver
metadata[
'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng'
self.logger.debug('[DONE]')
return metadata
def uploadShapefile(self, geoserver_url, workspace, storename, filename,
projection, secret_key, proxy_on):
self.logger.debug("Uploading shapefile" + filename + "...")
if (proxy_on.lower() == 'true'):
self.logger.debug("proxy set to on ....")
# TODO activate proxy_on method if the proxy in clowder works
return self.geoserver_manipulation_proxy_off(
geoserver_url, workspace, storename, filename, projection)
# return self.geoserver_manipulation_proxy_on(geoserver_url, workspace, storename, filename, projection, secret_key)
else:
return self.geoserver_manipulation_proxy_off(
geoserver_url, workspace, storename, filename, projection)
def geoserver_manipulation_proxy_off(self, geoserver_url, workspace,
storename, filename, projection):
self.logger.debug("start geoserver manipulation....")
# create workspace if not present
is_workspace = False
self.logger.debug("checking workspace %s" % workspace)
response_worksp = requests.get(self.restserver + 'workspaces/' +
workspace,
auth=(self.username, self.password))
if response_worksp.status_code != 200:
new_worksp = "<workspace><name>" + workspace + "</name></workspace>"
response_worksp = requests.post(
self.restserver + 'workspaces',
headers={"Content-type": "text/xml"},
auth=(self.username, self.password),
data=new_worksp)
if response_worksp.status_code == 201:
is_workspace = True
else:
is_workspace = True
if is_workspace:
url = self.restserver + "workspaces/" + workspace + "/datastores/" + storename + "/file.shp"
response = None
self.logger.debug("put file to geosever %s" % url)
with open(filename, 'rb') as f:
response = requests.put(
url,
headers={'content-type': 'application/zip'},
auth=(self.username, self.password),
data=f)
self.logger.debug(str(response.status_code) + " " + response.text)
if response.status_code != 201:
self.logger.debug("[DONE]")
return False
self.set_projection(storename, workspace, projection)
return True
else:
return False
def geoserver_manipulation_proxy_on(self, geoserver_url, workspace,
storename, filename, projection,
secret_key):
# create workspace if not present
is_workspace = False
# this is a direct method, if the proxy works, this should go through proxy
last_charactor = geoserver_url[-1]
if last_charactor == '/':
geoserver_rest = geoserver_url + 'rest'
else:
geoserver_rest = geoserver_url + '/rest'
response_worksp = requests.get(geoserver_rest + '/workspaces/' +
workspace + '?key=' + secret_key,
auth=(self.username, self.password))
if response_worksp.status_code != 200:
new_worksp = "<workspace><name>" + workspace + "</name></workspace>"
response_worksp = requests.post(
geoserver_rest + '/workspaces' + '?key=' + secret_key,
headers={"Content-type": "text/xml"},
auth=(self.username, self.password),
data=new_worksp)
if response_worksp.status_code == 201:
is_workspace = True
else:
is_workspace = True
if is_workspace:
url = geoserver_rest + "/workspaces/" + workspace + "/datastores/" + storename + "/file.shp"
response = None
with open(filename, 'rb') as f:
response = requests.put(
url + '?key=' + secret_key,
headers={'content-type': 'application/zip'},
data=f)
self.logger.debug(str(response.status_code) + " " + response.text)
if response.status_code != 201:
self.logger.debug("[DONE]")
return False
self.set_projection(storename, workspace, projection)
return True
else:
return False
def set_projection(self, storename, workspace, projection):
resource = self.getResourceByStoreName(storename, workspace)
if resource.projection == None:
self.logger.debug('Setting projection' + projection)
resource.projection = projection
self.catalog.save(resource)
self.logger.debug("[DONE]")
self.layerName = storename
def createThumbnail(self, workspace, storename, extent, width, height):
self.logger.debug('Creating Thumbnail ...')
layername = None
if self.layerName == None:
if self.layer == None:
self.logger.debug("getResourceByStoreName..")
resource = self.getResourceByStoreName(storename, workspace)
self.logger.debug("getLayerByResource ...")
layer = self.getLayerByResource(resource)
self.logger.debug("done getting layer name")
if layer == None:
self.logger.debug('No layer found [DONE]')
return metadata
else:
layername = layer.name
else:
self.logger.debug("layer instance found: no need to fetch")
layername = self.layer.name
self.layerName = self.layer.name
else:
self.logger.debug("layerName instance found: no need to fetch")
layername = self.layerName
#wmsLayerName = workspace+":"+layer.name
wmsLayerName = workspace + ":" + layername
url = self.wmsserver + "?request=GetMap&layers=" + wmsLayerName + "&bbox=" + extent + "&width=" + width + "&height=" + height + "&srs=EPSG:3857&format=image%2Fpng"
r = requests.get(url, stream=True)
path = os.path.join(self.tempDir, 'tmp.png')
if r.status_code == 200:
tmp = r.headers['content-disposition']
tmplist = tmp.split(';')
for t in tmplist:
if t.strip().find('filename=') != -1:
path = os.path.join(self.tempDir, t.strip().split('=')[1])
with open(path, 'wb') as f:
for chunk in r.iter_content():
f.write(chunk)
self.logger.debug('[DONE]')
return path
else:
self.logger.debug('can not create thumbnail [DONE]')
return ''
def __del__(self):
# delete the temp dir
if self.tempDir != None:
try:
import shutil
self.logger.debug("Deleting temp dir " + self.tempDir)
shutil.rmtree(self.tempDir)
self.logger.debug("Deleted Temp file")
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
def get_selected_raster(db,region,variable,date):
# logging.info(str(region)+','+str(variable)+','+str(date))
try:
# logging.info('Connecting to the database')
conn = psycopg2.connect("dbname={0} user={1} host={2} password={3}".format(db,cfg.connection['user'],cfg.connection['host'],cfg.connection['password']))
cur = conn.cursor()
storename = db+'_'+region+'_'+variable+'_'+date
cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password'],disable_ssl_certificate_validation=True)
try:
# logging.info('Check if the layer exists')
something = cat.get_store(storename,cfg.geoserver['workspace'])
if not something:
# logging.info('Layer doesnt exist')
print "No store"
raise Exception
else:
mean, stddev, min, max = get_vic_summary(db,region, variable, date)
# logging.info(str(mean)+str(stddev)+str(min)+str(max))
return storename, mean, stddev, min, max
except Exception as e:
# logging.info('Entering geoserver code')
# logging.error('Error at failed request ' + str(e))
try:
# logging.info('Starting the geoserver stuff')
sql = """SELECT ST_AsGDALRaster(rast, 'GTiff') as tiff FROM {0}.{1} WHERE id={2}""".format(region, variable, date)
cur.execute(sql)
data = cur.fetchall()
# logging.info(str(data))
mean, stddev, min, max = get_vic_summary(db,region, variable, date)
# logging.info('Work you piece ...')
rest_url = cfg.geoserver['rest_url']
# logging.info(str(rest_url))
if rest_url[-1] != "/":
rest_url = rest_url + '/'
headers = {
'Content-type': 'image/tiff',
}
request_url = '{0}workspaces/{1}/coveragestores/{2}/file.geotiff'.format(rest_url,
cfg.geoserver['workspace'],
storename) # Creating the rest url
# logging.info('Get the username and password')
user = cfg.geoserver['user']
password = cfg.geoserver['password']
# logging.info('Right before the put command')
requests.put(request_url,verify=False,headers=headers, data=data[0][0],
auth=(user, password)) # Creating the resource on the geoserver
# logging.info(request_url)
return storename, mean, stddev, min, max
except Exception as er:
# logging.info('Error at uplaoding tiff '+ str(e))
return str(er)+' This is while adding the raster layer.'
except Exception as err:
# logging.info(str(err) + ' This is generic catch all')
return str(err)+ ' This is the generic one'
def _register_cascaded_layers(service, owner=None):
"""
Register layers for a cascading WMS
"""
if service.type == "WMS" or service.type == "OWS":
cat = Catalog(settings.OGC_SERVER["default"]["LOCATION"] + "rest", _user, _password)
# Can we always assume that it is geonode?
# Should cascading layers have a separate workspace?
cascade_ws = cat.get_workspace(service.name)
if cascade_ws is None:
cascade_ws = cat.create_workspace(service.name, "cascade")
try:
store = cat.get_store(service.name, cascade_ws)
except Exception:
store = cat.create_wmsstore(service.name, cascade_ws)
cat.save(store)
wms = WebMapService(service.base_url)
layers = list(wms.contents)
count = 0
for layer in layers:
lyr = cat.get_resource(layer, store, cascade_ws)
if lyr is None:
if service.type in ["WMS", "OWS"]:
resource = cat.create_wmslayer(cascade_ws, store, layer)
elif service.type == "WFS":
resource = cat.create_wfslayer(cascade_ws, store, layer)
if resource:
bbox = resource.latlon_bbox
cascaded_layer, created = Layer.objects.get_or_create(
typename="%s:%s" % (cascade_ws.name, resource.name),
service=service,
defaults={
"name": resource.name,
"workspace": cascade_ws.name,
"store": store.name,
"storeType": store.resource_type,
"title": resource.title or "No title provided",
"abstract": resource.abstract or "No abstract provided",
"owner": None,
"uuid": str(uuid.uuid4()),
"bbox_x0": bbox[0],
"bbox_x1": bbox[1],
"bbox_y0": bbox[2],
"bbox_y1": bbox[3],
},
)
if created:
cascaded_layer.save()
if cascaded_layer is not None and cascaded_layer.bbox is None:
cascaded_layer._populate_from_gs(gs_resource=resource)
cascaded_layer.set_default_permissions()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service, typename=cascaded_layer.name
)
service_layer.layer = cascaded_layer
service_layer.title = (cascaded_layer.title,)
service_layer.description = (cascaded_layer.abstract,)
service_layer.styles = cascaded_layer.styles
service_layer.save()
count += 1
else:
logger.error("Resource %s from store %s could not be saved as layer" % (layer, store.name))
message = "%d Layers Registered" % count
return_dict = {"status": "ok", "msg": message}
return HttpResponse(json.dumps(return_dict), mimetype="application/json", status=200)
elif service.type == "WCS":
return HttpResponse("Not Implemented (Yet)", status=501)
else:
return HttpResponse("Invalid Service Type", status=400)
def calculate_yield(db,schema):
try:
conn = psycopg2.connect(
"dbname={0} user={1} host={2} password={3}".format(db, cfg.connection['user'], cfg.connection['host'],
cfg.connection['password']))
cur = conn.cursor()
storename = db+'_'+schema+'_agareas'
cat = Catalog(cfg.geoserver['rest_url'], username=cfg.geoserver['user'], password=cfg.geoserver['password'],disable_ssl_certificate_validation=True)
try:
print('Check if the layer exists')
something = cat.get_store(storename, cfg.geoserver['workspace'])
if not something:
print "No store"
raise Exception
else:
print 'Store exists'
except Exception as e:
print('Entering geoserver code')
temp_dir = tempfile.mkdtemp()
pg_sql = """SELECT * FROM {0}.agareas""".format(schema)
export_pg_table(temp_dir,storename,cfg.connection['host'],cfg.connection['user'],cfg.connection['password'],db,pg_sql)
target_zip = os.path.join(os.path.join(temp_dir,storename+'.zip'))
with zipfile.ZipFile(target_zip, 'w') as pg_zip:
for f in os.listdir(os.path.join(temp_dir)):
if '.zip' not in f:
f = os.path.join(temp_dir,f)
pg_zip.write(f,basename(f))
rest_url = cfg.geoserver['rest_url']
if rest_url[-1] != "/":
rest_url = rest_url + '/'
headers = {
'Content-type': 'application/zip',
}
request_url = '{0}workspaces/{1}/datastores/{2}/file.shp'.format(rest_url,
cfg.geoserver['workspace'],
storename) # Creating the rest url
user = cfg.geoserver['user']
password = cfg.geoserver['password']
requests.put(request_url, verify=False, headers=headers, data=open(target_zip,'rb'),
auth=(user, password)) # Creating the resource on the geoserver
if temp_dir is not None:
if os.path.exists(temp_dir):
print 'whooo'
#shutil.rmtree(temp_dir)
sql = """SELECT gid,avg(max) as max FROM(SELECT gid,ensemble,max(gwad) FROM {0}.dssat GROUP BY gid,ensemble ORDER BY gid,ensemble) as foo GROUP BY gid""".format(schema)
cur.execute(sql)
data = cur.fetchall()
data.sort()
conn.close()
return data,storename
except Exception as e:
print e
return e
class CatalogTests(unittest.TestCase):
def setUp(self):
self.cat = Catalog("http://localhost:8080/geoserver/rest")
def testAbout(self):
about_html = self.cat.about()
self.assertTrue(
'<html xmlns="http://www.w3.org/1999/xhtml"' in about_html)
def testGSVersion(self):
version = self.cat.gsversion()
pat = re.compile('\d\.\d(\.[\dx]|-SNAPSHOT)')
self.assertTrue(pat.match('2.2.x'))
self.assertTrue(pat.match('2.3.2'))
self.assertTrue(pat.match('2.3-SNAPSHOT'))
self.assertFalse(pat.match('2.3.y'))
self.assertFalse(pat.match('233'))
self.assertTrue(pat.match(version))
def testWorkspaces(self):
self.assertEqual(7, len(self.cat.get_workspaces()))
# marking out test since geoserver default workspace is not consistent
# self.assertEqual("cite", self.cat.get_default_workspace().name)
self.assertEqual("topp", self.cat.get_workspace("topp").name)
def testStores(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
self.assertEqual(9, len(self.cat.get_stores()))
self.assertEqual(2, len(self.cat.get_stores(topp)))
self.assertEqual(2, len(self.cat.get_stores(sf)))
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile", topp).name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("states_shapefile",
self.cat.get_store("states_shapefile").name)
self.assertEqual("sfdem", self.cat.get_store("sfdem", sf).name)
self.assertEqual("sfdem", self.cat.get_store("sfdem").name)
def testResources(self):
topp = self.cat.get_workspace("topp")
sf = self.cat.get_workspace("sf")
states = self.cat.get_store("states_shapefile", topp)
sfdem = self.cat.get_store("sfdem", sf)
self.assertEqual(19, len(self.cat.get_resources()))
self.assertEqual(1, len(self.cat.get_resources(states)))
self.assertEqual(5, len(self.cat.get_resources(workspace=topp)))
self.assertEqual(1, len(self.cat.get_resources(sfdem)))
self.assertEqual(6, len(self.cat.get_resources(workspace=sf)))
self.assertEqual("states",
self.cat.get_resource("states", states).name)
self.assertEqual("states",
self.cat.get_resource("states", workspace=topp).name)
self.assertEqual("states", self.cat.get_resource("states").name)
states = self.cat.get_resource("states")
fields = [
states.title, states.abstract, states.native_bbox,
states.latlon_bbox, states.projection, states.projection_policy
]
self.assertFalse(None in fields, str(fields))
self.assertFalse(len(states.keywords) == 0)
self.assertFalse(len(states.attributes) == 0)
self.assertTrue(states.enabled)
self.assertEqual("sfdem", self.cat.get_resource("sfdem", sfdem).name)
self.assertEqual("sfdem",
self.cat.get_resource("sfdem", workspace=sf).name)
self.assertEqual("sfdem", self.cat.get_resource("sfdem").name)
def testLayers(self):
expected = set([
"Arc_Sample", "Pk50095", "Img_Sample", "mosaic", "sfdem",
"bugsites", "restricted", "streams", "archsites", "roads",
"tasmania_roads", "tasmania_water_bodies",
"tasmania_state_boundaries", "tasmania_cities", "states",
"poly_landmarks", "tiger_roads", "poi", "giant_polygon"
])
actual = set(l.name for l in self.cat.get_layers())
missing = expected - actual
extras = actual - expected
message = "Actual layer list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
states = self.cat.get_layer("states")
self.assert_("states", states.name)
self.assert_(isinstance(states.resource, ResourceInfo))
self.assertEqual(set(s.name for s in states.styles),
set(['pophatch', 'polygon']))
self.assertEqual(states.default_style.name, "population")
def testLayerGroups(self):
expected = set(["tasmania", "tiger-ny", "spearfish"])
actual = set(l.name for l in self.cat.get_layergroups())
missing = expected - actual
extras = actual - expected
message = "Actual layergroup list did not match expected! (Extras: %s) (Missing: %s)" % (
extras, missing)
self.assert_(len(expected ^ actual) == 0, message)
tas = self.cat.get_layergroup("tasmania")
self.assert_("tasmania", tas.name)
self.assert_(isinstance(tas, LayerGroup))
self.assertEqual(tas.layers, [
'tasmania_state_boundaries', 'tasmania_water_bodies',
'tasmania_roads', 'tasmania_cities'
], tas.layers)
self.assertEqual(tas.styles, [None, None, None, None], tas.styles)
def testStyles(self):
self.assertEqual(20, len(self.cat.get_styles()))
self.assertEqual("population", self.cat.get_style("population").name)
self.assertEqual("popshade.sld",
self.cat.get_style("population").filename)
self.assertEqual("population",
self.cat.get_style("population").sld_name)
self.assert_(self.cat.get_style('non-existing-style') is None)
def testEscaping(self):
# GSConfig is inconsistent about using exceptions vs. returning None
# when a resource isn't found.
# But the basic idea is that none of them should throw HTTP errors from
# misconstructed URLS
self.cat.get_style("best style ever")
self.cat.get_workspace("best workspace ever")
try:
self.cat.get_store(workspace="best workspace ever",
name="best store ever")
self.fail('expected exception')
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever',
fre.message)
try:
self.cat.get_resource(workspace="best workspace ever",
store="best store ever",
name="best resource ever")
except FailedRequestError, fre:
self.assertEqual('No store found named: best store ever',
fre.message)
def _register_cascaded_service(url,
type,
name,
username,
password,
wms=None,
owner=None,
parent=None):
"""
Register a service as cascading WMS
"""
try:
service = Service.objects.get(base_url=url)
return_dict = {}
return_dict['service_id'] = service.pk
return_dict['msg'] = "This is an existing Service"
return HttpResponse(json.dumps(return_dict),
mimetype='application/json',
status=200)
except:
# TODO: Handle this error properly
pass
if wms is None:
wms = WebMapService(url)
# TODO: Make sure we are parsing all service level metadata
# TODO: Handle for setting ServiceProfiletRole
service = Service.objects.create(base_url=url,
type=type,
method='C',
name=name,
version=wms.identification.version,
title=wms.identification.title,
abstract=wms.identification.abstract,
online_resource=wms.provider.url,
owner=owner,
parent=parent)
service.keywords = ','.join(wms.identification.keywords)
service.save()
service.set_default_permissions()
if type in ['WMS', 'OWS']:
# Register the Service with GeoServer to be cascaded
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user, _password)
cascade_ws = cat.get_workspace(name)
if cascade_ws is None:
cascade_ws = cat.create_workspace(name,
"http://geonode.org/cascade")
# TODO: Make sure there isn't an existing store with that name, and
# deal with it if there is
try:
cascade_store = cat.get_store(name, cascade_ws)
except:
cascade_store = cat.create_wmsstore(name, cascade_ws, username,
password)
cascade_store.capabilitiesURL = url
cascade_store.type = "WMS"
cat.save(cascade_store)
available_resources = cascade_store.get_resources(available=True)
elif type == 'WFS':
# Register the Service with GeoServer to be cascaded
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user, _password)
# Can we always assume that it is geonode?
cascade_ws = cat.get_workspace(settings.CASCADE_WORKSPACE)
if cascade_ws is None:
cascade_ws = cat.create_workspace(settings.CASCADE_WORKSPACE,
"http://geonode.org/cascade")
try:
wfs_ds = cat.get_store(name, cascade_ws)
except:
wfs_ds = cat.create_datastore(name, cascade_ws)
connection_params = {
"WFSDataStoreFactory:MAXFEATURES": "0",
"WFSDataStoreFactory:TRY_GZIP": "true",
"WFSDataStoreFactory:PROTOCOL": "false",
"WFSDataStoreFactory:LENIENT": "true",
"WFSDataStoreFactory:TIMEOUT": "3000",
"WFSDataStoreFactory:BUFFER_SIZE": "10",
"WFSDataStoreFactory:ENCODING": "UTF-8",
"WFSDataStoreFactory:WFS_STRATEGY": "nonstrict",
"WFSDataStoreFactory:GET_CAPABILITIES_URL": url,
}
if username and password:
connection_params["WFSDataStoreFactory:USERNAME"] = username
connection_params["WFSDataStoreFactory:PASSWORD"] = password
wfs_ds.connection_parameters = connection_params
cat.save(wfs_ds)
available_resources = wfs_ds.get_resources(available=True)
# Save the Service record
service, created = Service.objects.get_or_create(type=type,
method='C',
base_url=url,
name=name,
owner=owner)
service.save()
service.set_default_permissions()
elif type == 'WCS':
return HttpResponse('Not Implemented (Yet)', status=501)
else:
return HttpResponse('Invalid Method / Type combo: ' +
'Only Cascaded WMS, WFS and WCS supported',
mimetype="text/plain",
status=400)
message = "Service %s registered" % service.name
return_dict = [{
'status': 'ok',
'msg': message,
'service_id': service.pk,
'service_name': service.name,
'service_title': service.title,
'available_layers': available_resources
}]
if settings.USE_QUEUE:
# Create a layer import job
WebServiceHarvestLayersJob.objects.get_or_create(service=service)
else:
_register_cascaded_layers(service)
return HttpResponse(json.dumps(return_dict),
mimetype='application/json',
status=200)
#!/usr/bin/env python
"""
gsconfig is a python library for manipulating a GeoServer instance via the GeoServer RESTConfig API.
The project is distributed under a MIT License .
"""
__author__ = "David Winslow"
__copyright__ = "Copyright 2012-2015 Boundless, Copyright 2010-2012 OpenPlans"
__license__ = "MIT"
from geoserver.catalog import Catalog
cat = Catalog("http://localhost:8080/geoserver/rest", "admin", "geoserver")
name = 'foo'
ds = cat.create_datastore(name)
ds.connection_parameters.update(host="localhost",
port="5432",
database="gis",
user="******",
passwd="",
dbtype="postgis")
cat.save(ds)
ds = cat.get_store(name)
components = \
dict((ext, "myfile." + ext) for ext in ["shp", "prj", "shx", "dbf"])
cat.add_data_to_store(ds, "mylayer", components)
def _register_cascaded_layers(service, owner=None):
"""
Register layers for a cascading WMS
"""
if service.type == 'WMS' or service.type == "OWS":
cat = Catalog(settings.OGC_SERVER['default']['LOCATION'] + "rest",
_user, _password)
# Can we always assume that it is geonode?
# Should cascading layers have a separate workspace?
cascade_ws = cat.get_workspace(service.name)
if cascade_ws is None:
cascade_ws = cat.create_workspace(service.name, 'cascade')
try:
store = cat.get_store(service.name, cascade_ws)
except Exception:
store = cat.create_wmsstore(service.name, cascade_ws)
cat.save(store)
wms = WebMapService(service.base_url)
layers = list(wms.contents)
count = 0
for layer in layers:
lyr = cat.get_resource(layer, store, cascade_ws)
if lyr is None:
if service.type in ["WMS", "OWS"]:
resource = cat.create_wmslayer(cascade_ws, store, layer)
elif service.type == "WFS":
resource = cat.create_wfslayer(cascade_ws, store, layer)
if resource:
bbox = resource.latlon_bbox
cascaded_layer, created = Layer.objects.get_or_create(
typename="%s:%s" % (cascade_ws.name, resource.name),
service=service,
defaults={
"name": resource.name,
"workspace": cascade_ws.name,
"store": store.name,
"storeType": store.resource_type,
"title": resource.title or 'No title provided',
"abstract": resource.abstract
or 'No abstract provided',
"owner": None,
"uuid": str(uuid.uuid4()),
"bbox_x0": bbox[0],
"bbox_x1": bbox[1],
"bbox_y0": bbox[2],
"bbox_y1": bbox[3],
})
if created:
cascaded_layer.save()
if cascaded_layer is not None and cascaded_layer.bbox is None:
cascaded_layer._populate_from_gs(
gs_resource=resource)
cascaded_layer.set_default_permissions()
service_layer, created = ServiceLayer.objects.get_or_create(
service=service, typename=cascaded_layer.name)
service_layer.layer = cascaded_layer
service_layer.title = cascaded_layer.title,
service_layer.description = cascaded_layer.abstract,
service_layer.styles = cascaded_layer.styles
service_layer.save()
count += 1
else:
logger.error(
"Resource %s from store %s could not be saved as layer"
% (layer, store.name))
message = "%d Layers Registered" % count
return_dict = {'status': 'ok', 'msg': message}
return HttpResponse(json.dumps(return_dict),
mimetype='application/json',
status=200)
elif service.type == 'WCS':
return HttpResponse('Not Implemented (Yet)', status=501)
else:
return HttpResponse('Invalid Service Type', status=400)
