def _addGroup(layer_tree):
layers = []
children = layer_tree.children()
children.reverse() # GS and QGIS have opposite ordering
for child in children:
if isinstance(child, QgsLayerTreeLayer):
child_layer = child.layer()
_, out_name = lyr_utils.getLayerTitleAndName(child_layer)
if child_layer.id() in self.layer_ids:
layers.append(out_name)
elif isinstance(child, QgsLayerTreeGroup):
subgroup = _addGroup(child)
if subgroup is not None:
layers.append(subgroup)
if layers:
title, name = lyr_utils.getLayerTitleAndName(layer_tree)
return {
"name": name,
"title": layer_tree.customProperty("wmsTitle", title),
"abstract":
layer_tree.customProperty("wmsAbstract", title),
"layers": layers
}
else:
return None
def viewWms(self, layer_id):
server = manager.getGeodataServer(self.comboGeodataServer.currentText())
if not server:
return
layer = getLayerById(layer_id)
_, name = getLayerTitleAndName(layer)
bbox = layer.extent()
if bbox.isEmpty():
bbox.grow(1)
self.previewWebService(server, [name], bbox, layer.crs().authid())
def publishStyle(self, layer):
lyr_title, lyr_name = lyr_utils.getLayerTitleAndName(layer)
export_layer = lyr_utils.getExportableLayer(layer, lyr_name)
style_filename = tempFileInSubFolder(lyr_name + ".zip")
warnings = saveLayerStyleAsZippedSld(export_layer, style_filename)
for w in warnings:
self.logWarning(w)
self.logInfo(f"Style for layer '{layer.name()}' exported as ZIP file to '{style_filename}'")
self._publishStyle(lyr_name, style_filename)
self._published_layers.add(layer)
return style_filename
def unpublishData(self, layer_id):
server = manager.getGeodataServer(self.comboGeodataServer.currentText())
if not server:
return
layer = getLayerById(layer_id)
_, name = getLayerTitleAndName(layer)
if server.deleteLayer(name):
# Deletion was successful: silently try to remove style (should have been removed already)
server.deleteStyle(name)
# Mark layer as deleted
self.updateLayerIsDataPublished(layer_id, None)
def saveMetadata(layer, mefFilename=None, apiUrl=None, wms=None, wfs=None, layerName=None):
uuid = uuidForLayer(layer)
_, safe_name = getLayerTitleAndName(layer)
filename = tempFileInSubFolder(safe_name + ".qmd")
layer.saveNamedMetadata(filename)
thumbnail = _saveLayerThumbnail(layer)
apiUrl = apiUrl or ""
transformedFilename = _transformMetadata(filename, uuid, apiUrl, wms, wfs, layerName or safe_name)
mefFilename = mefFilename or tempFileInSubFolder(uuid + ".mef")
_createMef(uuid, transformedFilename, mefFilename, thumbnail)
return mefFilename
def publishLayer(self, layer, fields=None):
lyr_title, safe_name = lyr_utils.getLayerTitleAndName(layer)
if layer.type() == layer.VectorLayer:
if layer.featureCount() == 0:
self.logError(f"Layer '{lyr_title}' contains zero features and cannot be published")
return
if layer.dataProvider().name() == "postgres" and self.useOriginalDataSource:
try:
from geocatbridge.servers.models.postgis import PostgisServer
except (ImportError, ModuleNotFoundError):
raise Exception(self.translate(getAppName(), "Cannot find or import PostgisServer class"))
else:
uri = QgsDataSourceUri(layer.source())
db = PostgisServer(
"temp", uri.authConfigId(), uri.host(), uri.port(), uri.schema(), uri.database())
self._publishVectorLayerFromPostgis(layer, db)
elif self.storage != GeoserverStorage.POSTGIS_BRIDGE:
src_path, src_name, src_ext = lyr_utils.getLayerSourceInfo(layer)
filename = self._exported_layers.get(src_path)
if not filename:
if self.storage == GeoserverStorage.POSTGIS_GEOSERVER:
shp_name = exportLayer(layer, fields, to_shapefile=True, force=True, logger=self)
basename = os.path.splitext(shp_name)[0]
filename = basename + ".zip"
with ZipFile(filename, 'w') as z:
for ext in (".shp", ".shx", ".prj", ".dbf"):
filetozip = basename + ext
z.write(filetozip, arcname=os.path.basename(filetozip))
else:
filename = exportLayer(layer, fields, logger=self)
self._exported_layers[src_path] = filename
if self.storage == GeoserverStorage.FILE_BASED:
self._publishVectorLayerFromFile(layer, filename)
else:
self._publishVectorLayerFromFileToPostgis(layer, filename)
elif self.storage == GeoserverStorage.POSTGIS_BRIDGE:
db = manager.getServer(self.postgisdb)
if not db:
raise Exception(self.translate(getAppName(), "Cannot find the selected PostGIS database"))
db.importLayer(layer, fields)
self._publishVectorLayerFromPostgis(layer, db)
elif layer.type() == layer.RasterLayer:
if layer.source() not in self._exported_layers:
path = exportLayer(layer, fields, logger=self)
self._exported_layers[layer.source()] = path
filename = self._exported_layers[layer.source()]
self._publishRasterLayer(filename, safe_name)
self._clearCache()
def viewAllWms(self):
server = manager.getGeodataServer(self.comboGeodataServer.currentText())
if not server:
return
bbox = QgsRectangle()
crs = iface.mapCanvas().mapSettings().destinationCrs()
names = []
for layer in self.publishableLayers:
if not self.isDataPublished.get(layer.id()):
continue
_, name = getLayerTitleAndName(layer)
names.append(name)
xform = QgsCoordinateTransform(layer.crs(), crs, QgsProject().instance())
extent = xform.transform(layer.extent())
bbox.combineExtentWith(extent)
self.previewWebService(server, names, bbox, crs.authid())
def run(self):
try:
os.makedirs(self.folder, exist_ok=True)
for i, id_ in enumerate(self.layer_ids):
if self.isCanceled():
return False
self.setProgress(i * 100 / len(self.layer_ids))
layer = self.getLayerById(id_)
name, safe_name = lyr_utils.getLayerTitleAndName(layer)
if self.export_symbology:
style_filename = os.path.join(self.folder,
safe_name + "_style.zip")
self.stepStarted.emit(id_, SYMBOLOGY)
saveLayerStyleAsZippedSld(layer, style_filename)
self.stepFinished.emit(id_, SYMBOLOGY)
else:
self.stepSkipped.emit(id_, SYMBOLOGY)
if self.export_data:
ext = ".gpkg" if layer.type(
) == layer.VectorLayer else ".tif"
layer_filename = os.path.join(self.folder, safe_name + ext)
self.stepStarted.emit(id_, DATA)
exportLayer(layer,
self.field_map[id_],
path=layer_filename,
force=True,
logger=self)
self.stepFinished.emit(id_, DATA)
else:
self.stepSkipped.emit(id_, DATA)
if self.export_metadata:
metadata_filename = os.path.join(
self.folder, safe_name + "_metadata.zip")
self.stepStarted.emit(id_, METADATA)
saveMetadata(layer, metadata_filename)
self.stepFinished.emit(id_, METADATA)
else:
self.stepSkipped.emit(id_, METADATA)
return True
except Exception:
self.exception = traceback.format_exc()
return False
def _publishVectorLayerFromFile(self, layer, filename):
self.logInfo(f"Publishing layer from file: {filename}")
title, name = lyr_utils.getLayerTitleAndName(layer)
is_data_uploaded = filename in self._uploaded_data
if not is_data_uploaded:
with open(filename, "rb") as f:
self._deleteDatastore(name)
url = f"{self.apiUrl}/workspaces/{self.workspace}/datastores/{name}/file.gpkg?update=overwrite"
self.request(url, "put", f.read())
conn = sqlite3.connect(filename)
cursor = conn.cursor()
cursor.execute("""SELECT table_name FROM gpkg_geometry_columns""") # noqa
tablename = cursor.fetchall()[0][0]
self._uploaded_data[filename] = (name, tablename)
dataset_name, geoserver_layer_name = self._uploaded_data[filename]
url = f"{self.apiUrl}/workspaces/{self.workspace}/datastores/{dataset_name}/featuretypes/{geoserver_layer_name}.json" # noqa
r = self.request(url)
ft = r.json()
ft["featureType"]["name"] = name
ft["featureType"]["title"] = title
ext = layer.extent()
ft["featureType"]["nativeBoundingBox"] = {
"minx": round(ext.xMinimum(), 5),
"maxx": round(ext.xMaximum(), 5),
"miny": round(ext.yMinimum(), 5),
"maxy": round(ext.yMaximum(), 5),
"srs": layer.crs().authid()
}
if is_data_uploaded:
url = f"{self.apiUrl}/workspaces/{self.workspace}/datastores/{dataset_name}/featuretypes"
self.request(url, "post", ft)
else:
self.request(url, "put", ft)
self.logInfo(f"Successfully created feature type from GeoPackage file '{filename}'")
self._setLayerStyle(name)
def _publishVectorLayerFromFileToPostgis(self, layer, filename):
self.logInfo(f"Publishing layer '{layer.name()}' from file '{filename}'...")
datastore = self.createPostgisDatastore()
title, ft_name = lyr_utils.getLayerTitleAndName(layer)
source_name = os.path.splitext(os.path.basename(filename))[0]
# Create a new import
body = {
"import": {
"targetStore": {
"dataStore": {
"name": datastore
}
},
"targetWorkspace": {
"workspace": {
"name": self.workspace
}
}
}
}
url = f"{self.apiUrl}/imports.json"
ret = self.request(url, "post", body)
# Create a new task and upload ZIP
self.logInfo("Uploading layer data...")
import_id = ret.json()["import"]["id"]
zipname = os.path.basename(filename)
url = f"{self.apiUrl}/imports/{import_id}/tasks/{zipname}"
with open(filename, "rb") as f:
ret = self.request(url, method="put", files={zipname: (zipname, f, 'application/octet-stream')})
# Reassign PostGIS datastore as target (just to be sure)
task_id = ret.json()["task"]["id"]
body = {
"dataStore": {
"name": datastore
}
}
url = f"{self.apiUrl}/imports/{import_id}/tasks/{task_id}/target.json"
self.request(url, "put", body)
del ret
# Start import execution
self.logInfo(f"Starting Importer task for layer '{ft_name}'...")
url = f"{self.apiUrl}/imports/{import_id}"
self.request(url, method="post")
# Get the import result (error message and target layer name)
import_err, tmp_name = self._getImportResult(import_id, task_id)
if import_err:
self.logError(f"Failed to publish QGIS layer '{title}' as '{ft_name}'.\n\n{import_err}")
return
self._uploaded_data[filename] = (datastore, source_name)
# Get the created feature type
self.logInfo("Checking if feature type creation was successful...")
url = f"{self.apiUrl}/workspaces/{self.workspace}/datastores/{datastore}/featuretypes/{tmp_name}.json"
try:
ret = self.request(url + "?quietOnNotFound=true")
except HTTPError as e:
# Something unexpected happened: failure cannot be retrieved from import task,
# so the user should check the GeoServer logs to find out what caused it.
if e.response.status_code == 404:
self.logError(f"Failed to publish QGIS layer '{title}' as '{ft_name}' due to an unknown error.\n"
"Please check the GeoServer logs.")
return
raise
# Modify the feature type descriptions, but leave the name in tact to avoid db schema mismatches
self.logInfo("Fixing feature type properties...")
ft = ret.json()
ft["featureType"]["nativeName"] = tmp_name # name given by Importer extension
ft["featureType"]["originalName"] = source_name # source file name
ft["featureType"]["title"] = title # layer name as displayed in QGIS
self.request(url, "put", ft)
self.logInfo(f"Successfully created feature type from file '{filename}'")
# Fix layer style reference and remove unwanted global style
self.logInfo("Performing style cleanup...")
try:
self._fixLayerStyle(tmp_name, ft_name)
except HTTPError as e:
self.logWarning(f"Failed to clean up layer styles: {e}")
else:
self.logInfo(f"Successfully published layer '{title}'")
def exportLayer(layer,
fields=None,
to_shapefile=False,
path=None,
force=False,
logger=None):
logger = logger or feedback
filepath, _, ext = lyr_utils.getLayerSourceInfo(layer)
lyr_name, safe_name = lyr_utils.getLayerTitleAndName(layer)
fields = fields or []
if layer.type() == layer.VectorLayer:
if to_shapefile and (force or layer.fields().count() != len(fields)
or ext != EXT_SHAPEFILE):
# Export with Shapefile extension
ext = EXT_SHAPEFILE
elif force or ext != EXT_GEOPACKAGE or layer.fields().count() != len(fields) \
or not isSingleTableGpkg(filepath):
# Export with GeoPackage extension
ext = EXT_GEOPACKAGE
else:
# No need to export
logger.logInfo(
f"No need to export layer {lyr_name} stored at {filepath}")
return filepath
# Perform GeoPackage or Shapefile export
attrs = [
i for i, f in enumerate(layer.fields())
if len(fields) == 0 or f.name() in fields
]
output = path or tempFileInSubFolder(safe_name + ext)
encoding = "UTF-8"
driver = "ESRI Shapefile" if ext == EXT_SHAPEFILE else "GPKG"
options = None
if hasattr(QgsVectorFileWriter, 'SaveVectorOptions'):
# QGIS v3.x has the SaveVectorOptions object
options = QgsVectorFileWriter.SaveVectorOptions()
options.fileEncoding = encoding
options.attributes = attrs
options.driverName = driver
# Make sure that we are using the latest (non-deprecated) write method
if hasattr(QgsVectorFileWriter, 'writeAsVectorFormatV3'):
# Use writeAsVectorFormatV3 for QGIS versions >= 3.20 to avoid DeprecationWarnings
result = QgsVectorFileWriter.writeAsVectorFormatV3(
layer, output, QgsCoordinateTransformContext(),
options) # noqa
elif hasattr(QgsVectorFileWriter, 'writeAsVectorFormatV2'):
# Use writeAsVectorFormatV2 for QGIS versions >= 3.10.3 to avoid DeprecationWarnings
result = QgsVectorFileWriter.writeAsVectorFormatV2(
layer, output, QgsCoordinateTransformContext(),
options) # noqa
else:
# Use writeAsVectorFormat for QGIS versions < 3.10.3 for backwards compatibility
result = QgsVectorFileWriter.writeAsVectorFormat(
layer,
output,
fileEncoding=encoding,
attributes=attrs,
driverName=driver) # noqa
# Check if first item in result tuple is an error code
if result[0] == QgsVectorFileWriter.NoError:
logger.logInfo(f"Layer {lyr_name} exported to {output}")
else:
# Dump the result tuple as-is when there are errors (the tuple size depends on the QGIS version)
logger.logError(
f"Layer {lyr_name} failed to export.\n\tResult object: {str(result)}"
)
return output
else:
# Export raster
if force or not filepath.lower().endswith("tif"):
output = path or tempFileInSubFolder(safe_name + ".tif")
writer = QgsRasterFileWriter(output)
writer.setOutputFormat("GTiff")
writer.writeRaster(layer.pipe(), layer.width(), layer.height(),
layer.extent(), layer.crs())
del writer
logger.logInfo(f"Layer {lyr_name} exported to {output}")
return output
else:
logger.logInfo(
f"No need to export layer {lyr_name} stored at {filepath}")
return filepath
def isDataOnServer(self, layer):
server = manager.getGeodataServer(self.comboGeodataServer.currentText())
if not server:
return False
_, name = getLayerTitleAndName(layer)
return server.layerExists(name)
def run(self):
def publishLayer(lyr, lyr_name):
fields = None
if lyr.type() == lyr.VectorLayer:
fields = [
_name
for _name, publish in self.field_map[lyr.id()].items()
if publish
]
self.geodata_server.publishLayer(lyr, fields)
if self.metadata_server is not None:
metadata_uuid = uuidForLayer(lyr)
md_url = self.metadata_server.metadataUrl(metadata_uuid)
self.geodata_server.setLayerMetadataLink(lyr_name, md_url)
try:
validator = QgsNativeMetadataValidator()
# FIXME: remove or improve this
# DONOTALLOW = 0
ALLOW = 1
ALLOWONLYDATA = 2
allow_without_md = ALLOW # pluginSetting("allowWithoutMetadata")
if self.geodata_server is not None:
self.geodata_server.prepareForPublishing(self.only_symbology)
self.results = {}
for i, layer_id in enumerate(self.layer_ids):
if self.isCanceled():
return False
warnings, errors = [], []
self.setProgress(i * 100 / len(self.layer_ids))
layer = lyr_utils.getLayerById(layer_id)
name, safe_name = lyr_utils.getLayerTitleAndName(layer)
if not lyr_utils.hasValidLayerName(layer):
try:
warnings.append(
f"Layer name '{name}' contains characters that may cause issues"
)
except UnicodeError:
warnings.append(
"Layer name contains characters that may cause issues"
)
md_valid, _ = validator.validate(layer.metadata())
if self.geodata_server is not None:
self.geodata_server.resetLogIssues()
# Publish style
self.stepStarted.emit(layer_id, SYMBOLOGY)
try:
self.geodata_server.publishStyle(layer)
except:
errors.append(traceback.format_exc())
self.stepFinished.emit(layer_id, SYMBOLOGY)
if self.only_symbology:
# Skip data publish if "only symbology" was checked
self.stepSkipped.emit(layer_id, DATA)
else:
# Publish data
self.stepStarted.emit(layer_id, DATA)
try:
if md_valid or allow_without_md in (ALLOW,
ALLOWONLYDATA):
publishLayer(layer, safe_name)
else:
self.stepStarted.emit(layer_id, DATA)
if md_valid or allow_without_md in (
ALLOW, ALLOWONLYDATA):
publishLayer(layer, safe_name)
else:
self.geodata_server.logError(
f"Layer '{name}' has invalid metadata. "
f"Layer was not published")
self.stepFinished.emit(layer_id, DATA)
except:
errors.append(traceback.format_exc())
self.stepFinished.emit(layer_id, DATA)
else:
# No geodata server selected: skip layer data and symbology
self.stepSkipped.emit(layer_id, SYMBOLOGY)
self.stepSkipped.emit(layer_id, DATA)
if self.metadata_server is not None:
# User selected metadata server: publish metadata
try:
self.metadata_server.resetLogIssues()
if md_valid or allow_without_md == ALLOW:
wms = None
wfs = None
full_name = None
if self.geodata_server is not None:
full_name = self.geodata_server.fullLayerName(
safe_name)
wms = self.geodata_server.getWmsUrl()
if layer.type() == layer.VectorLayer:
wfs = self.geodata_server.getWfsUrl()
self.autofillMetadata(layer)
self.stepStarted.emit(layer_id, METADATA)
self.metadata_server.publishLayerMetadata(
layer, wms, wfs, full_name)
self.stepFinished.emit(layer_id, METADATA)
else:
self.metadata_server.logError(
f"Layer '{name}' has invalid metadata. "
f"Metadata was not published")
except:
errors.append(traceback.format_exc())
else:
self.stepSkipped.emit(layer_id, METADATA)
# Collect all layer-specific errors and warnings (if any)
if self.geodata_server is not None:
w, e = self.geodata_server.getLogIssues()
warnings.extend(w)
errors.extend(e)
if self.metadata_server is not None:
w, e = self.metadata_server.getLogIssues()
warnings.extend(w)
errors.extend(e)
self.results[name] = (set(warnings), set(errors))
# Create layer groups (if any)
if self.geodata_server is not None:
self.stepStarted.emit(None, GROUPS)
try:
# FIXME (did this ever work?)
self.geodata_server.createGroups(self._layerGroups(),
self.layer_ids)
except Exception as err:
# TODO: figure out where to properly put a warning or error message for this
feedback.logError(f"Could not create layer groups: {err}")
finally:
try:
# Call closePublishing(): for GeoServer, this will set up vector tiles, if enabled
self.geodata_server.closePublishing()
except Exception as err:
feedback.logError(
f"Failed to finalize publish task: {err}")
self.stepFinished.emit(None, GROUPS)
else:
self.stepSkipped.emit(None, GROUPS)
return True
except Exception:
self.exc_type, _, _ = sys.exc_info()
self.exception = traceback.format_exc()
return False