def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
if isinstance(catalog, PKICatalog):
self.client = PKIClient(catalog.service_url, catalog.key, catalog.cert, catalog.ca_cert)
else:
self.client = Client(str(catalog.service_url), catalog.username, catalog.password)
def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
if isinstance(catalog, PKICatalog):
self.client = PKIClient(catalog.service_url, catalog.key,
catalog.cert, catalog.ca_cert)
else:
self.client = Client(str(catalog.service_url), catalog.username,
catalog.password)
def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
self.client = Client(str(catalog.service_url), catalog.username, catalog.password)
class OGCatalog(object):
'''
This class is a wrapper for a catalog object, with convenience methods to use it with QGIS layers
'''
def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
self.client = Client(str(catalog.service_url), catalog.username, catalog.password)
def clean(self):
self.cleanUnusedStyles()
self.cleanUnusedResources()
def cleanUnusedStyles(self):
'''cleans styles that are not used by any layer'''
usedStyles = set()
styles = self.catalog.get_styles()
layers = self.catalog.get_layers()
for layer in layers:
usedStyles.add(layer.default_style.name)
usedStyles.update([s.name for s in layer.styles if s is not None])
toDelete = [s for s in styles if s.name not in usedStyles]
for style in toDelete:
style.catalog.delete(style, purge = True)
def cleanUnusedResources(self):
'''cleans resources that are not published through any layer in the catalog'''
usedResources = set()
resources = self.catalog.get_resources()
layers = self.catalog.get_layers()
for layer in layers:
usedResources.add(layer.resource.name)
toDelete = [r for r in resources if r.name not in usedResources]
for resource in toDelete:
resource.catalog.delete(resource)
for store in self.catalog.get_stores():
if len(store.get_resources()) == 0:
self.catalog.delete(store)
def consolidateStyles(self):
'''
Deletes styles that are redundant and just keeps one copy of them
in the catalog, configuring the corresponding layers to use that copy
'''
used = {}
allstyles = self.catalog.get_styles()
for style in allstyles:
sld = style.sld_body.replace("<sld:Name>%s</sld:Name>" % style.name, "")
if sld in used.keys():
used[sld].append(style)
else:
used[sld] = [style]
for sld, styles in used.iteritems():
if len(styles) == 1:
continue
#find the layers that use any of the secondary styles in the list, and make them use the first one
styleNames = [s.name for s in styles[1:]]
layers = self.catalog.get_layers()
for layer in layers:
changed = False
if layer.default_style.name in styleNames:
layer.default_style = styles[0]
changed = True
alternateStyles = layer.styles
newAlternateStyles = set()
for alternateStyle in alternateStyles:
if alternateStyle.name in styleNames:
newAlternateStyles.add(styles[0])
else:
newAlternateStyles.add(alternateStyle)
newAlternateStyles = list(newAlternateStyles)
if newAlternateStyles != alternateStyles:
layer.styles = newAlternateStyles
changed = True
if changed:
self.catalog.save(layer)
def publishStyle(self, layer, overwrite = True, name = None):
'''
Publishes the style of a given layer style in the specified catalog. If the overwrite parameter is True,
it will overwrite a style with that name in case it exists
'''
if isinstance(layer, basestring):
layer = layers.resolveLayer(layer)
sld = getGsCompatibleSld(layer)
if sld is not None:
name = name if name is not None else layer.name()
name = name.replace(" ", "_")
self.catalog.create_style(name, sld, overwrite)
return sld
def getDataFromLayer(self, layer):
'''
Returns the data corresponding to a given layer, ready to be passed to the
method in the Catalog class for uploading to the server.
If needed, it performs an export to ensure that the file format is supported
by the upload API to be used for import. In that case, the data returned
will point to the exported copy of the data, not the original data source
'''
if layer.type() == layer.RasterLayer:
data = exporter.exportRasterLayer(layer)
else:
filename = exporter.exportVectorLayer(layer)
basename, extension = os.path.splitext(filename)
data = {
'shp': basename + '.shp',
'shx': basename + '.shx',
'dbf': basename + '.dbf',
'prj': basename + '.prj'
}
return data
def _publishExisting(self, layer, workspace, overwrite):
connName = self.getConnectionNameFromLayer(layer)
uri = QgsDataSourceURI(layer.dataProvider().dataSourceUri())
store = createPGFeatureStore(self.catalog,
connName,
workspace = workspace,
overwrite = overwrite,
host = uri.host(),
database = uri.database(),
schema = uri.schema(),
port = uri.port(),
user = uri.username(),
passwd = uri.password())
self.catalog.publish_featuretype(uri.table(), store, layer.crs().authid())
def _uploadRest(self, layer, workspace, overwrite, name):
if layer.type() == layer.RasterLayer:
path = self.getDataFromLayer(layer)
self.catalog.create_coveragestore(name,
path,
workspace=workspace,
overwrite=overwrite)
elif layer.type() == layer.VectorLayer:
path = self.getDataFromLayer(layer)
self.catalog.create_featurestore(name,
path,
workspace=workspace,
overwrite=overwrite)
def _uploadImporter(self, layer, workspace, overwrite, name):
# @todo - more richness needed to allow ingestion into target store
# versus just publishing the layer to a workspace as a shapefile
path = self.getDataFromLayer(layer)
if isinstance(path, dict):
if 'shp' in path:
path = path['shp']
else:
raise Exception('Unexpected condition : %s', path.keys())
session = self.client.upload(path)
if not session.tasks:
raise Exception('Geoserver is not able to process the uploaded data')
if len(session.tasks) != 1:
# this probably shouldn't happen but just in case
raise Exception('Unexpected condition')
# set workspace if needed (network trip)
# limitation in setting name as importer interprets this as a request
# to use an existing store by name
if workspace:
session.tasks[0].set_target(workspace=workspace.name)
if overwrite:
session.tasks[0].set_update_mode('REPLACE')
session.commit()
def upload(self, layer, workspace=None, overwrite=True, name=None):
'''uploads the specified layer'''
if isinstance(layer, basestring):
layer = layers.resolveLayer(layer)
name = name if name is not None else layer.name()
title = name
name = name.replace(" ", "_")
settings = QSettings()
restApi = bool(settings.value("/OpenGeo/Settings/GeoServer/UseRestApi", True, bool))
if layer.type() not in (layer.RasterLayer, layer.VectorLayer):
msg = layer.name() + ' is not a valid raster or vector layer'
raise Exception(msg)
provider = layer.dataProvider()
try:
if provider.name() == 'postgres':
self._publishExisting(layer, workspace, overwrite)
elif restApi:
self._uploadRest(layer, workspace, overwrite, name)
else:
self._uploadImporter(layer, workspace, overwrite, name)
except UploadError, e:
msg = ('Could not save the layer %s, there was an upload '
'error: %s' % (layer.name(), str(e)))
e.args = (msg,)
raise
except ConflictingDataError, e:
# A datastore of this name already exists
msg = ('GeoServer reported a conflict creating a store with name %s: '
'"%s". This should never happen because a brand new name '
'should have been generated. But since it happened, '
'try renaming the file or deleting the store in '
'GeoServer.' % (layer.name(), str(e)))
e.args = (msg,)
raise e
def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
self.client = Client(str(catalog.service_url), catalog.username,
catalog.password)
class OGCatalog(object):
'''
This class is a wrapper for a catalog object, with convenience methods to use it with QGIS layers
'''
def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
self.client = Client(str(catalog.service_url), catalog.username,
catalog.password)
def clean(self):
self.cleanUnusedStyles()
self.cleanUnusedResources()
def cleanUnusedStyles(self):
'''cleans styles that are not used by any layer'''
usedStyles = set()
styles = self.catalog.get_styles()
layers = self.catalog.get_layers()
for layer in layers:
usedStyles.add(layer.default_style.name)
usedStyles.update([s.name for s in layer.styles if s is not None])
toDelete = [s for s in styles if s.name not in usedStyles]
for style in toDelete:
style.catalog.delete(style, purge=True)
def cleanUnusedResources(self):
'''cleans resources that are not published through any layer in the catalog'''
usedResources = set()
resources = self.catalog.get_resources()
layers = self.catalog.get_layers()
for layer in layers:
usedResources.add(layer.resource.name)
toDelete = [r for r in resources if r.name not in usedResources]
for resource in toDelete:
resource.catalog.delete(resource)
for store in self.catalog.get_stores():
if len(store.get_resources()) == 0:
self.catalog.delete(store)
def consolidateStyles(self):
'''
Deletes styles that are redundant and just keeps one copy of them
in the catalog, configuring the corresponding layers to use that copy
'''
used = {}
allstyles = self.catalog.get_styles()
for style in allstyles:
sld = style.sld_body.replace(
"<sld:Name>%s</sld:Name>" % style.name, "")
if sld in used.keys():
used[sld].append(style)
else:
used[sld] = [style]
for sld, styles in used.iteritems():
if len(styles) == 1:
continue
#find the layers that use any of the secondary styles in the list, and make them use the first one
styleNames = [s.name for s in styles[1:]]
layers = self.catalog.get_layers()
for layer in layers:
changed = False
if layer.default_style.name in styleNames:
layer.default_style = styles[0]
changed = True
alternateStyles = layer.styles
newAlternateStyles = set()
for alternateStyle in alternateStyles:
if alternateStyle.name in styleNames:
newAlternateStyles.add(styles[0])
else:
newAlternateStyles.add(alternateStyle)
newAlternateStyles = list(newAlternateStyles)
if newAlternateStyles != alternateStyles:
layer.styles = newAlternateStyles
changed = True
if changed:
self.catalog.save(layer)
def publishStyle(self, layer, overwrite=True, name=None):
'''
Publishes the style of a given layer style in the specified catalog. If the overwrite parameter is True,
it will overwrite a style with that name in case it exists
'''
if isinstance(layer, basestring):
layer = layers.resolveLayer(layer)
sld = getGsCompatibleSld(layer)
if sld is not None:
name = name if name is not None else layer.name()
name = name.replace(" ", "_")
self.catalog.create_style(name, sld, overwrite)
return sld
def getDataFromLayer(self, layer):
'''
Returns the data corresponding to a given layer, ready to be passed to the
method in the Catalog class for uploading to the server.
If needed, it performs an export to ensure that the file format is supported
by the upload API to be used for import. In that case, the data returned
will point to the exported copy of the data, not the original data source
'''
if layer.type() == layer.RasterLayer:
data = exporter.exportRasterLayer(layer)
else:
filename = exporter.exportVectorLayer(layer)
basename, extension = os.path.splitext(filename)
data = {
'shp': basename + '.shp',
'shx': basename + '.shx',
'dbf': basename + '.dbf',
'prj': basename + '.prj'
}
return data
def _publishExisting(self, layer, workspace, overwrite):
connName = self.getConnectionNameFromLayer(layer)
uri = QgsDataSourceURI(layer.dataProvider().dataSourceUri())
store = createPGFeatureStore(self.catalog,
connName,
workspace=workspace,
overwrite=overwrite,
host=uri.host(),
database=uri.database(),
schema=uri.schema(),
port=uri.port(),
user=uri.username(),
passwd=uri.password())
self.catalog.publish_featuretype(uri.table(), store,
layer.crs().authid())
def _uploadRest(self, layer, workspace, overwrite, name):
if layer.type() == layer.RasterLayer:
path = self.getDataFromLayer(layer)
self.catalog.create_coveragestore(name,
path,
workspace=workspace,
overwrite=overwrite)
elif layer.type() == layer.VectorLayer:
path = self.getDataFromLayer(layer)
self.catalog.create_featurestore(name,
path,
workspace=workspace,
overwrite=overwrite)
def _uploadImporter(self, layer, workspace, overwrite, name):
# @todo - more richness needed to allow ingestion into target store
# versus just publishing the layer to a workspace as a shapefile
path = self.getDataFromLayer(layer)
if isinstance(path, dict):
if 'shp' in path:
path = path['shp']
else:
raise Exception('Unexpected condition : %s', path.keys())
session = self.client.upload(path)
if not session.tasks:
raise Exception(
'Geoserver is not able to process the uploaded data')
if len(session.tasks) != 1:
# this probably shouldn't happen but just in case
raise Exception('Unexpected condition')
# set workspace if needed (network trip)
# limitation in setting name as importer interprets this as a request
# to use an existing store by name
if workspace:
session.tasks[0].set_target(workspace=workspace.name)
if overwrite:
session.tasks[0].set_update_mode('REPLACE')
session.commit()
def upload(self, layer, workspace=None, overwrite=True, name=None):
'''uploads the specified layer'''
if isinstance(layer, basestring):
layer = layers.resolveLayer(layer)
name = name if name is not None else layer.name()
title = name
name = name.replace(" ", "_")
settings = QSettings()
restApi = bool(
settings.value("/OpenGeo/Settings/GeoServer/UseRestApi", True,
bool))
if layer.type() not in (layer.RasterLayer, layer.VectorLayer):
msg = layer.name() + ' is not a valid raster or vector layer'
raise Exception(msg)
provider = layer.dataProvider()
try:
if provider.name() == 'postgres':
self._publishExisting(layer, workspace, overwrite)
elif restApi:
self._uploadRest(layer, workspace, overwrite, name)
else:
self._uploadImporter(layer, workspace, overwrite, name)
except UploadError, e:
msg = ('Could not save the layer %s, there was an upload '
'error: %s' % (layer.name(), str(e)))
e.args = (msg, )
raise
except ConflictingDataError, e:
# A datastore of this name already exists
msg = (
'GeoServer reported a conflict creating a store with name %s: '
'"%s". This should never happen because a brand new name '
'should have been generated. But since it happened, '
'try renaming the file or deleting the store in '
'GeoServer.' % (layer.name(), str(e)))
e.args = (msg, )
raise e
class OGCatalog(object):
'''
This class is a wrapper for a catalog object, with convenience methods to use it with QGIS layers
'''
def __init__(self, catalog):
self.catalog = catalog
#we also create a Client object pointing to the same url
if isinstance(catalog, PKICatalog):
self.client = PKIClient(catalog.service_url, catalog.key, catalog.cert, catalog.ca_cert)
else:
self.client = Client(str(catalog.service_url), catalog.username, catalog.password)
def clean(self):
self.cleanUnusedStyles()
self.cleanUnusedResources()
def cleanUnusedStyles(self):
'''cleans styles that are not used by any layer'''
usedStyles = set()
styles = self.catalog.get_styles()
layers = self.catalog.get_layers()
groups = self.catalog.get_layergroups()
for layer in layers:
usedStyles.add(layer.default_style.name)
usedStyles.update([s.name for s in layer.styles if s is not None])
for group in groups:
usedStyles.update([s for s in group.styles if s is not None])
toDelete = [s for s in styles if s.name not in usedStyles]
for style in toDelete:
style.catalog.delete(style, purge = True)
def cleanUnusedResources(self):
'''cleans resources that are not published through any layer in the catalog'''
usedResources = set()
resources = self.catalog.get_resources()
layers = self.catalog.get_layers()
for layer in layers:
usedResources.add(layer.resource.name)
toDelete = [r for r in resources if r.name not in usedResources]
for resource in toDelete:
resource.catalog.delete(resource)
for store in self.catalog.get_stores():
if len(store.get_resources()) == 0:
self.catalog.delete(store)
def consolidateStyles(self):
'''
Deletes styles that are redundant and just keeps one copy of them
in the catalog, configuring the corresponding layers to use that copy
'''
used = {}
allstyles = self.catalog.get_styles()
for style in allstyles:
sld = style.sld_body.replace("<sld:Name>%s</sld:Name>" % style.name, "")
if sld in used.keys():
used[sld].append(style)
else:
used[sld] = [style]
for sld, styles in used.iteritems():
if len(styles) == 1:
continue
#find the layers that use any of the secondary styles in the list, and make them use the first one
styleNames = [s.name for s in styles[1:]]
layers = self.catalog.get_layers()
for layer in layers:
changed = False
if layer.default_style.name in styleNames:
layer.default_style = styles[0]
changed = True
alternateStyles = layer.styles
newAlternateStyles = set()
for alternateStyle in alternateStyles:
if alternateStyle.name in styleNames:
newAlternateStyles.add(styles[0])
else:
newAlternateStyles.add(alternateStyle)
newAlternateStyles = list(newAlternateStyles)
if newAlternateStyles != alternateStyles:
layer.styles = newAlternateStyles
changed = True
if changed:
self.catalog.save(layer)
def publishStyle(self, layer, overwrite = True, name = None):
'''
Publishes the style of a given layer style in the specified catalog. If the overwrite parameter is True,
it will overwrite a style with that name in case it exists
'''
if isinstance(layer, basestring):
layer = layers.resolveLayer(layer)
sld = getGsCompatibleSld(layer)
if sld is not None:
name = name if name is not None else layer.name()
name = name.replace(" ", "_")
self.catalog.create_style(name, sld, overwrite)
return sld
def getDataFromLayer(self, layer):
'''
Returns the data corresponding to a given layer, ready to be passed to the
method in the Catalog class for uploading to the server.
If needed, it performs an export to ensure that the file format is supported
by the upload API to be used for import. In that case, the data returned
will point to the exported copy of the data, not the original data source
'''
if layer.type() == layer.RasterLayer:
data = exporter.exportRasterLayer(layer)
else:
filename = exporter.exportVectorLayer(layer)
basename, extension = os.path.splitext(filename)
data = {
'shp': basename + '.shp',
'shx': basename + '.shx',
'dbf': basename + '.dbf',
'prj': basename + '.prj'
}
return data
def _publishExisting(self, layer, workspace, overwrite,
name, storename=None):
uri = QgsDataSourceURI(layer.dataProvider().dataSourceUri())
# check for table.name conflict in existing layer names where the
# table.name is not the same as the user-chosen layer name,
# i.e. unintended overwrite
resource = self.catalog.get_resource(uri.table())
if resource is not None and uri.table() != name:
raise Exception("QGIS PostGIS layer has table name conflict with "
"existing GeoServer layer name: {0}\n"
"You may need to rename GeoServer layer name."
.format(uri.table()))
conname = self.getConnectionNameFromLayer(layer)
storename = xmlNameFixUp(storename or conname)
if not xmlNameIsValid(storename):
raise Exception("Database connection name is invalid XML and can "
"not be auto-fixed: {0} -> {1}"
.format(conname, storename))
if not uri.username():
raise Exception("GeoServer requires database connection's username "
"to be defined")
store = createPGFeatureStore(self.catalog,
storename,
workspace = workspace,
overwrite = overwrite,
host = uri.host(),
database = uri.database(),
schema = uri.schema(),
port = uri.port(),
user = uri.username(),
passwd = uri.password())
if store is not None:
rscname = name if uri.table() != name else uri.table()
grpswlyr = []
if overwrite:
# TODO: How do we honor *unchecked* user setting of
# "Delete resource when deleting layer" here?
# Is it an issue, if overwrite is expected?
# We will soon have two layers with slightly different names,
# a temp based upon table.name, the other possibly existing
# layer with the same custom name, which may belong to group(s).
# If so, remove existing layer from any layer group, before
# continuing on with layer delete and renaming of new feature
# type layer to custom name, then add new resultant layer back
# to any layer groups the existing layer belonged to. Phew!
flyr = self.catalog.get_layer(rscname)
if flyr is not None:
grpswlyr = groupsWithLayer(self.catalog, flyr)
if grpswlyr:
removeLayerFromGroups(self.catalog, flyr, grpswlyr)
self.catalog.delete(flyr)
# TODO: What about when the layer name is the same, but the
# underlying db connection/store has changed? Not an issue?
# The layer is deleted, which is correct, but the original
# db store and feature type will not be changed. A conflict?
frsc = store.get_resources(name=rscname)
if frsc is not None:
self.catalog.delete(frsc)
# for dbs the name has to be the table name, initially
ftype = self.catalog.publish_featuretype(uri.table(), store,
layer.crs().authid())
# once table-based feature type created, switch name to user-chosen
if ftype.name != rscname:
ftype.dirty["name"] = rscname
self.catalog.save(ftype)
# now re-add to any previously assigned-to layer groups
if overwrite and grpswlyr:
ftype = self.catalog.get_resource(rscname)
if ftype:
addLayerToGroups(self.catalog, ftype, grpswlyr,
workspace=workspace)
def _uploadRest(self, layer, workspace, overwrite, name):
if layer.type() == layer.RasterLayer:
path = self.getDataFromLayer(layer)
self.catalog.create_coveragestore(name,
path,
workspace=workspace,
overwrite=overwrite)
elif layer.type() == layer.VectorLayer:
path = self.getDataFromLayer(layer)
self.catalog.create_featurestore(name,
path,
workspace=workspace,
overwrite=overwrite)
def _uploadImporter(self, layer, workspace, overwrite, name):
# @todo - more richness needed to allow ingestion into target store
# versus just publishing the layer to a workspace as a shapefile
path = self.getDataFromLayer(layer)
if isinstance(path, dict):
if 'shp' in path:
path = path['shp']
else:
raise Exception('Unexpected condition : %s', path.keys())
session = self.client.upload(path)
if not session.tasks:
raise Exception('Geoserver is not able to process the uploaded data')
if len(session.tasks) != 1:
# this probably shouldn't happen but just in case
raise Exception('Unexpected condition')
# set workspace if needed (network trip)
# limitation in setting name as importer interprets this as a request
# to use an existing store by name
if workspace:
session.tasks[0].set_target(workspace=workspace.name)
if overwrite:
session.tasks[0].set_update_mode('REPLACE')
session.commit()
def upload(self, layer, workspace=None, overwrite=True, name=None,
storename=None, title=None):
'''uploads the specified layer'''
if isinstance(layer, basestring):
layer = layers.resolveLayer(layer)
name = name or layer.name()
title = title or name # name is usually xml-fixed-up by now
name = xmlNameFixUp(name)
if storename is not None:
storename = xmlNameFixUp(storename)
settings = QSettings()
restApi = bool(settings.value("/OpenGeo/Settings/GeoServer/UseRestApi", True, bool))
if layer.type() not in (layer.RasterLayer, layer.VectorLayer):
msg = layer.name() + ' is not a valid raster or vector layer'
raise Exception(msg)
provider = layer.dataProvider()
try:
if provider.name() == 'postgres':
self._publishExisting(layer, workspace, overwrite, name,
storename=storename)
elif restApi:
self._uploadRest(layer, workspace, overwrite, name)
else:
self._uploadImporter(layer, workspace, overwrite, name)
except UploadError, e:
msg = ('Could not save the layer %s, there was an upload '
'error: %s' % (layer.name(), str(e)))
e.args = (msg,)
raise
except ConflictingDataError, e:
# A datastore of this name already exists
msg = ('GeoServer reported a conflict creating a store with name %s: '
'"%s". This should never happen because a brand new name '
'should have been generated. But since it happened, '
'try renaming the file or deleting the store in '
'GeoServer.' % (layer.name(), str(e)))
e.args = (msg,)
raise e
