def validateMetadata(self):
if self.currentLayer is None:
return
self.storeMetadata()
validator = QgsNativeMetadataValidator()
result, errors = validator.validate(self.metadata[self.currentLayer.id()])
if result:
html = f"<p>{self.tr('No validation errors')}</p>"
else:
issues = "".join(f"<li><b>{e.section}</b>: {e.note}</li>" for e in errors)
html = f"<p>{self.tr('The following issues were found')}:<ul>{issues}</ul></p>"
self.showHtmlMessage("Metadata validation", html)
def validateMetadata(self):
if self.currentLayer is None:
return
self.storeMetadata()
validator = QgsNativeMetadataValidator()
result, errors = validator.validate(self.metadata[self.currentLayer])
if result:
txt = self.tr("No validation errors")
else:
txt = self.tr(
"The following issues were found:") + "<br>" + "<br>".join([
"<b>%s</b>:%s" % (err.section, err.note) for err in errors
])
dlg = QgsMessageOutput.createMessageOutput()
dlg.setTitle(self.tr("Metadata validation"))
dlg.setMessage(txt, QgsMessageOutput.MessageHtml)
dlg.showMessage()
def testValidateNative(self): # spellok
"""
Test validating metadata against QGIS native schema
"""
m = self.createTestMetadata()
v = QgsNativeMetadataValidator()
res, list = v.validate(m)
self.assertTrue(res)
self.assertFalse(list)
# corrupt metadata piece by piece...
m = self.createTestMetadata()
m.setIdentifier('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'identifier')
m = self.createTestMetadata()
m.setLanguage('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'language')
m = self.createTestMetadata()
m.setType('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'type')
m = self.createTestMetadata()
m.setTitle('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'title')
m = self.createTestMetadata()
m.setAbstract('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'abstract')
m = self.createTestMetadata()
m.setLicenses([])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'license')
m = self.createTestMetadata()
m.setCrs(QgsCoordinateReferenceSystem())
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'crs')
m = self.createTestMetadata()
m.setContacts([])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'contacts')
m = self.createTestMetadata()
m.setLinks([])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
m = self.createTestMetadata()
m.setKeywords({'': ['kw1', 'kw2']})
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'keywords')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
m.setKeywords({'AA': []})
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'keywords')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
e = m.extent()
se = e.spatialExtents()[0]
se.extentCrs = QgsCoordinateReferenceSystem()
e.setSpatialExtents([se])
m.setExtent(e)
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'extent')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
e = m.extent()
se = e.spatialExtents()[0]
se.bounds = QgsBox3d(1, 1, 0, 1, 2)
e.setSpatialExtents([se])
m.setExtent(e)
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'extent')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
c = m.contacts()[0]
c.name = ''
m.setContacts([c])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'contacts')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
l = m.links()[0]
l.name = ''
m.setLinks([l])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
l = m.links()[0]
l.type = ''
m.setLinks([l])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
l = m.links()[0]
l.url = ''
m.setLinks([l])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
self.assertEqual(list[0].identifier, 0)
def testValidateNative(self): # spellok
"""
Test validating metadata against QGIS native schema
"""
m = self.createTestMetadata()
v = QgsNativeMetadataValidator()
res, list = v.validate(m)
self.assertTrue(res)
self.assertFalse(list)
# corrupt metadata piece by piece...
m = self.createTestMetadata()
m.setIdentifier('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'identifier')
m = self.createTestMetadata()
m.setLanguage('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'language')
m = self.createTestMetadata()
m.setType('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'type')
m = self.createTestMetadata()
m.setTitle('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'title')
m = self.createTestMetadata()
m.setAbstract('')
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'abstract')
m = self.createTestMetadata()
m.setLicenses([])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'license')
m = self.createTestMetadata()
m.setCrs(QgsCoordinateReferenceSystem())
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'crs')
m = self.createTestMetadata()
m.setContacts([])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'contacts')
m = self.createTestMetadata()
m.setLinks([])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
m = self.createTestMetadata()
m.setKeywords({'': ['kw1', 'kw2']})
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'keywords')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
m.setKeywords({'AA': []})
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'keywords')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
e = m.extent()
se = e.spatialExtents()[0]
se.extentCrs = QgsCoordinateReferenceSystem()
e.setSpatialExtents([se])
m.setExtent(e)
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'extent')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
e = m.extent()
se = e.spatialExtents()[0]
se.bounds = QgsBox3d(1, 1, 0, 1, 2)
e.setSpatialExtents([se])
m.setExtent(e)
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'extent')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
c = m.contacts()[0]
c.name = ''
m.setContacts([c])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'contacts')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
l = m.links()[0]
l.name = ''
m.setLinks([l])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
l = m.links()[0]
l.type = ''
m.setLinks([l])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
self.assertEqual(list[0].identifier, 0)
m = self.createTestMetadata()
l = m.links()[0]
l.url = ''
m.setLinks([l])
res, list = v.validate(m)
self.assertFalse(res)
self.assertEqual(list[0].section, 'links')
self.assertEqual(list[0].identifier, 0)
def run(self):
try:
validator = QgsNativeMetadataValidator()
DONOTALLOW = 0
ALLOW = 1
ALLOWONLYDATA = 2
allowWithoutMetadata = ALLOW #pluginSetting("allowWithoutMetadata")
if self.geodataServer is not None:
self.geodataServer.prepareForPublishing(self.onlySymbology)
self.results = {}
for i, name in enumerate(self.layers):
if self.isCanceled():
return False
warnings, errors = [], []
self.setProgress(i * 100 / len(self.layers))
layer = self.layerFromName(name)
warnings.extend(self.validateLayer(layer))
validates, _ = validator.validate(layer.metadata())
validates = True
if self.geodataServer is not None:
try:
self.geodataServer.resetLog()
self.stepStarted.emit(name, SYMBOLOGY)
self.geodataServer.publishStyle(layer)
self.stepFinished.emit(name, SYMBOLOGY)
except:
self.stepFinished.emit(name, SYMBOLOGY)
errors.append(traceback.format_exc())
try:
if self.onlySymbology:
self.stepSkipped.emit(name, DATA)
else:
self.stepStarted.emit(name, DATA)
if validates or allowWithoutMetadata in [
ALLOW, ALLOWONLYDATA
]:
fields = None
if layer.type() == layer.VectorLayer:
fields = [
name for name, publish in
self.fields[layer].items() if publish
]
self.geodataServer.publishLayer(layer, fields)
if self.metadataServer is not None:
metadataUuid = uuidForLayer(layer)
url = self.metadataServer.metadataUrl(
metadataUuid)
self.geodataServer.setLayerMetadataLink(
name, url)
else:
self.geodataServer.logError(
self.
tr("Layer '%s' has invalid metadata. Layer was not published"
) % layer.name())
self.stepFinished.emit(name, DATA)
except:
self.stepFinished.emit(name, DATA)
errors.append(traceback.format_exc())
else:
self.stepSkipped.emit(name, SYMBOLOGY)
self.stepSkipped.emit(name, DATA)
if self.metadataServer is not None:
try:
self.metadataServer.resetLog()
if validates or allowWithoutMetadata == ALLOW:
if self.geodataServer is not None:
fullName = self.geodataServer.fullLayerName(
layer.name())
wms = self.geodataServer.layerWmsUrl(
layer.name())
if layer.type() == layer.VectorLayer:
wfs = self.geodataServer.layerWfsUrl()
else:
wfs = None
else:
wms = None
wfs = None
fullName = None
self.autofillMetadata(layer)
self.stepStarted.emit(name, METADATA)
self.metadataServer.publishLayerMetadata(
layer, wms, wfs, fullName)
self.stepFinished.emit(name, METADATA)
else:
self.metadataServer.logError(
self.
tr("Layer '%s' has invalid metadata. Metadata was not published"
) % layer.name())
except:
errors.append(traceback.format_exc())
else:
self.stepSkipped.emit(name, METADATA)
if self.geodataServer is not None:
w, e = self.geodataServer.loggedInfo()
warnings.extend(w)
errors.extend(e)
if self.metadataServer is not None:
w, e = self.metadataServer.loggedInfo()
warnings.extend(w)
errors.extend(e)
self.results[name] = (set(warnings), set(errors))
if self.geodataServer is not None:
self.stepStarted.emit(None, GROUPS)
groups = self._layerGroups(self.layers)
try:
self.geodataServer.createGroups(groups)
except:
#TODO: figure out where to put a warning or error message for this
pass
finally:
self.geodataServer.closePublishing()
self.stepFinished.emit(None, GROUPS)
else:
self.stepSkipped.emit(None, GROUPS)
return True
except Exception as e:
self.exceptiontype, _, _ = sys.exc_info()
self.exception = traceback.format_exc()
return False
def run(self):
try:
validator = QgsNativeMetadataValidator()
DONOTALLOW = 0
ALLOW = 1
ALLOWONLYDATA = 2
allowWithoutMetadata = ALLOW #pluginSetting("allowWithoutMetadata")
if self.geodataServer is not None:
self.geodataServer.prepareForPublishing(self.onlySymbology)
self.results = {}
warnings, errors = [], []
for i, name in enumerate(self.layers):
if self.isCanceled():
return False
self.setProgress(i * 100 / len(self.layers))
try:
layer = self.layerFromName(name)
validates, _ = validator.validate(layer.metadata())
validates = True
if self.geodataServer is not None:
self.geodataServer.resetLog()
if self.onlySymbology:
self.geodataServer.publishStyle(layer)
else:
if validates or allowWithoutMetadata in [
ALLOW, ALLOWONLYDATA
]:
fields = None
if layer.type() == layer.VectorLayer:
fields = [
name for name, publish in
self.fields[layer].items() if publish
]
self.geodataServer.publishLayer(layer, fields)
if self.metadataServer is not None:
metadataUuid = uuidForLayer(layer)
url = self.metadataServer.metadataUrl(
metadataUuid)
self.geodataServer.setLayerMetadataLink(
name, url)
else:
self.geodataServer.logError(
self.
tr("Layer '%s' has invalid metadata. Layer was not published"
) % layer.name())
if self.metadataServer is not None:
self.metadataServer.resetLog()
if validates or allowWithoutMetadata == ALLOW:
if self.geodataServer is not None:
wms = self.geodataServer.layerWmsUrl(
layer.name())
else:
wms = None
self.autofillMetadata(layer)
self.metadataServer.publishLayerMetadata(
layer, wms)
else:
self.metadataServer.logError(
self.
tr("Layer '%s' has invalid metadata. Metadata was not published"
) % layer.name())
except:
errors.append(traceback.format_exc())
if self.geodataServer is not None:
w, e = self.geodataServer.loggedInfo()
warnings.extend(w)
errors.extend(e)
if self.metadataServer is not None:
w, e = self.metadataServer.loggedInfo()
warnings.extend(w)
errors.extend(e)
self.results[name] = (set(warnings), errors)
if self.geodataServer is not None:
groups = self._layerGroups(self.layers)
self.geodataServer.createGroups(groups)
self.geodataServer.closePublishing()
return True
except Exception as e:
self.exception = traceback.format_exc()
return False
def run(self):
def publishLayer(lyr, lyr_name):
fields = None
if lyr.type() == lyr.VectorLayer:
fields = [
_name
for _name, publish in self.field_map[lyr.id()].items()
if publish
]
self.geodata_server.publishLayer(lyr, fields)
if self.metadata_server is not None:
metadata_uuid = uuidForLayer(lyr)
md_url = self.metadata_server.metadataUrl(metadata_uuid)
self.geodata_server.setLayerMetadataLink(lyr_name, md_url)
try:
validator = QgsNativeMetadataValidator()
# FIXME: remove or improve this
# DONOTALLOW = 0
ALLOW = 1
ALLOWONLYDATA = 2
allow_without_md = ALLOW # pluginSetting("allowWithoutMetadata")
if self.geodata_server is not None:
self.geodata_server.prepareForPublishing(self.only_symbology)
self.results = {}
for i, layer_id in enumerate(self.layer_ids):
if self.isCanceled():
return False
warnings, errors = [], []
self.setProgress(i * 100 / len(self.layer_ids))
layer = lyr_utils.getLayerById(layer_id)
name, safe_name = lyr_utils.getLayerTitleAndName(layer)
if not lyr_utils.hasValidLayerName(layer):
try:
warnings.append(
f"Layer name '{name}' contains characters that may cause issues"
)
except UnicodeError:
warnings.append(
"Layer name contains characters that may cause issues"
)
md_valid, _ = validator.validate(layer.metadata())
if self.geodata_server is not None:
self.geodata_server.resetLogIssues()
# Publish style
self.stepStarted.emit(layer_id, SYMBOLOGY)
try:
self.geodata_server.publishStyle(layer)
except:
errors.append(traceback.format_exc())
self.stepFinished.emit(layer_id, SYMBOLOGY)
if self.only_symbology:
# Skip data publish if "only symbology" was checked
self.stepSkipped.emit(layer_id, DATA)
else:
# Publish data
self.stepStarted.emit(layer_id, DATA)
try:
if md_valid or allow_without_md in (ALLOW,
ALLOWONLYDATA):
publishLayer(layer, safe_name)
else:
self.stepStarted.emit(layer_id, DATA)
if md_valid or allow_without_md in (
ALLOW, ALLOWONLYDATA):
publishLayer(layer, safe_name)
else:
self.geodata_server.logError(
f"Layer '{name}' has invalid metadata. "
f"Layer was not published")
self.stepFinished.emit(layer_id, DATA)
except:
errors.append(traceback.format_exc())
self.stepFinished.emit(layer_id, DATA)
else:
# No geodata server selected: skip layer data and symbology
self.stepSkipped.emit(layer_id, SYMBOLOGY)
self.stepSkipped.emit(layer_id, DATA)
if self.metadata_server is not None:
# User selected metadata server: publish metadata
try:
self.metadata_server.resetLogIssues()
if md_valid or allow_without_md == ALLOW:
wms = None
wfs = None
full_name = None
if self.geodata_server is not None:
full_name = self.geodata_server.fullLayerName(
safe_name)
wms = self.geodata_server.getWmsUrl()
if layer.type() == layer.VectorLayer:
wfs = self.geodata_server.getWfsUrl()
self.autofillMetadata(layer)
self.stepStarted.emit(layer_id, METADATA)
self.metadata_server.publishLayerMetadata(
layer, wms, wfs, full_name)
self.stepFinished.emit(layer_id, METADATA)
else:
self.metadata_server.logError(
f"Layer '{name}' has invalid metadata. "
f"Metadata was not published")
except:
errors.append(traceback.format_exc())
else:
self.stepSkipped.emit(layer_id, METADATA)
# Collect all layer-specific errors and warnings (if any)
if self.geodata_server is not None:
w, e = self.geodata_server.getLogIssues()
warnings.extend(w)
errors.extend(e)
if self.metadata_server is not None:
w, e = self.metadata_server.getLogIssues()
warnings.extend(w)
errors.extend(e)
self.results[name] = (set(warnings), set(errors))
# Create layer groups (if any)
if self.geodata_server is not None:
self.stepStarted.emit(None, GROUPS)
try:
# FIXME (did this ever work?)
self.geodata_server.createGroups(self._layerGroups(),
self.layer_ids)
except Exception as err:
# TODO: figure out where to properly put a warning or error message for this
feedback.logError(f"Could not create layer groups: {err}")
finally:
try:
# Call closePublishing(): for GeoServer, this will set up vector tiles, if enabled
self.geodata_server.closePublishing()
except Exception as err:
feedback.logError(
f"Failed to finalize publish task: {err}")
self.stepFinished.emit(None, GROUPS)
else:
self.stepSkipped.emit(None, GROUPS)
return True
except Exception:
self.exc_type, _, _ = sys.exc_info()
self.exception = traceback.format_exc()
return False
def run(self):
def publishLayer(lyr, lyr_name):
fields = None
if lyr.type() == lyr.VectorLayer:
fields = [_name for _name, publish in self.fields[lyr].items() if publish]
self.geodata_server.publishLayer(lyr, fields)
if self.metadata_server is not None:
metadata_uuid = uuidForLayer(lyr)
md_url = self.metadata_server.metadataUrl(metadata_uuid)
self.geodata_server.setLayerMetadataLink(lyr_name, md_url)
try:
validator = QgsNativeMetadataValidator()
# DONOTALLOW = 0
ALLOW = 1
ALLOWONLYDATA = 2
allow_without_md = ALLOW # pluginSetting("allowWithoutMetadata")
if self.geodata_server is not None:
self.geodata_server.prepareForPublishing(self.only_symbology)
qgs_layers = {}
self.results = {}
for i, name in enumerate(self.layers):
if self.isCanceled():
return False
warnings, errors = [], []
self.setProgress(i * 100 / len(self.layers))
layer = self.layerFromName(name)
qgs_layers[name] = layer
_, safe_name = layerUtils.getLayerTitleAndName(layer)
if not layerUtils.hasValidLayerName(layer):
try:
warnings.append(f"Layer name '{name}' contains characters that might cause issues")
except UnicodeError:
warnings.append("Layer name contains characters that might cause issues")
md_valid, _ = validator.validate(layer.metadata())
if self.geodata_server is not None:
self.geodata_server.resetLog()
# Publish style
self.stepStarted.emit(name, SYMBOLOGY)
try:
self.geodata_server.publishStyle(layer)
except:
errors.append(traceback.format_exc())
self.stepFinished.emit(name, SYMBOLOGY)
if self.only_symbology:
self.stepSkipped.emit(name, DATA)
continue
# Publish data
self.stepStarted.emit(name, DATA)
try:
if validates or allow_without_md in (ALLOW, ALLOWONLYDATA):
publishLayer(layer, safe_name)
else:
self.stepStarted.emit(name, DATA)
if validates or allow_without_md in (ALLOW, ALLOWONLYDATA):
publishLayer(layer, safe_name)
else:
self.geodata_server.logError(f"Layer '{name}' has invalid metadata. "
f"Layer was not published")
self.stepFinished.emit(name, DATA)
except:
errors.append(traceback.format_exc())
self.stepFinished.emit(name, DATA)
else:
self.stepSkipped.emit(name, SYMBOLOGY)
self.stepSkipped.emit(name, DATA)
if self.metadata_server is not None:
try:
self.metadata_server.resetLog()
if md_valid or allow_without_md == ALLOW:
wms = None
wfs = None
full_name = None
if self.geodata_server is not None:
full_name = self.geodata_server.fullLayerName(safe_name)
wms = self.geodata_server.layerWmsUrl()
if layer.type() == layer.VectorLayer:
wfs = self.geodata_server.layerWfsUrl()
self.autofillMetadata(layer)
self.stepStarted.emit(name, METADATA)
self.metadata_server.publishLayerMetadata(layer, wms, wfs, full_name)
self.stepFinished.emit(name, METADATA)
else:
self.metadata_server.logError(f"Layer '{name}' has invalid metadata. "
f"Metadata was not published")
except:
errors.append(traceback.format_exc())
else:
self.stepSkipped.emit(name, METADATA)
if self.geodata_server is not None:
w, e = self.geodata_server.getLogIssues()
warnings.extend(w)
errors.extend(e)
if self.metadata_server is not None:
w, e = self.metadata_server.getLogIssues()
warnings.extend(w)
errors.extend(e)
self.results[name] = (set(warnings), set(errors))
if self.geodata_server is not None:
self.stepStarted.emit(None, GROUPS)
groups = self._layerGroups(self.layers)
try:
self.geodata_server.createGroups(groups, qgs_layers)
except Exception:
# TODO: figure out where to put a warning or error message for this
pass
finally:
self.stepFinished.emit(None, GROUPS)
else:
self.stepSkipped.emit(None, GROUPS)
return True
except Exception:
self.exc_type, _, _ = sys.exc_info()
self.exception = traceback.format_exc()
return False
